From 5005212bec654a0adcde5c9cb511e12c0926a3e6 Mon Sep 17 00:00:00 2001 From: syeopite <70992037+syeopite@users.noreply.github.com> Date: Mon, 30 Aug 2021 14:58:24 +0000 Subject: [PATCH] Extract feed routes (#2269) * Extract feed routes from invidious.cr * Removes the deprecated route for /feed/top * Deprecate /view_all_playlist & use /feed/playlists * Move feed views into their own directory * Add haltf method to halt current route context * Change status_code + return blocks to use haltf * Set appropriate response headers for RSS routes --- src/invidious.cr | 438 +----------------- src/invidious/helpers/macros.cr | 9 + src/invidious/routes/feeds.cr | 431 +++++++++++++++++ src/invidious/routes/misc.cr | 2 +- src/invidious/routes/playlists.cr | 27 +- src/invidious/views/{ => feeds}/history.ecr | 0 .../playlists.ecr} | 0 src/invidious/views/{ => feeds}/popular.ecr | 0 .../views/{ => feeds}/subscriptions.ecr | 0 src/invidious/views/{ => feeds}/trending.ecr | 0 src/invidious/views/playlist.ecr | 2 +- src/invidious/views/preferences.ecr | 2 +- 12 files changed, 462 insertions(+), 449 deletions(-) create mode 100644 src/invidious/routes/feeds.cr rename src/invidious/views/{ => feeds}/history.ecr (100%) rename src/invidious/views/{view_all_playlists.ecr => feeds/playlists.ecr} (100%) rename src/invidious/views/{ => feeds}/popular.ecr (100%) rename src/invidious/views/{ => feeds}/subscriptions.ecr (100%) rename src/invidious/views/{ => feeds}/trending.ecr (100%) diff --git a/src/invidious.cr b/src/invidious.cr index 402e8974..f497a527 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -349,7 +349,6 @@ Invidious::Routing.get "/redirect", Invidious::Routes::Misc, :cross_instance_red Invidious::Routing.get "/embed/", Invidious::Routes::Embed, :redirect Invidious::Routing.get "/embed/:id", Invidious::Routes::Embed, :show -Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Playlists, :index Invidious::Routing.get "/create_playlist", Invidious::Routes::Playlists, :new Invidious::Routing.post "/create_playlist", Invidious::Routes::Playlists, :create Invidious::Routing.get "/subscribe_playlist", Invidious::Routes::Playlists, :subscribe @@ -374,6 +373,24 @@ Invidious::Routing.get "/preferences", Invidious::Routes::PreferencesRoute, :sho Invidious::Routing.post "/preferences", Invidious::Routes::PreferencesRoute, :update Invidious::Routing.get "/toggle_theme", Invidious::Routes::PreferencesRoute, :toggle_theme +# Feeds +Invidious::Routing.get "/view_all_playlists", Invidious::Routes::Feeds, :view_all_playlists_redirect +Invidious::Routing.get "/feed/playlists", Invidious::Routes::Feeds, :playlists +Invidious::Routing.get "/feed/popular", Invidious::Routes::Feeds, :popular +Invidious::Routing.get "/feed/trending", Invidious::Routes::Feeds, :trending +Invidious::Routing.get "/feed/subscriptions", Invidious::Routes::Feeds, :subscriptions +Invidious::Routing.get "/feed/history", Invidious::Routes::Feeds, :history + +# RSS Feeds +Invidious::Routing.get "/feed/channel/:ucid", Invidious::Routes::Feeds, :rss_channel +Invidious::Routing.get "/feed/private", Invidious::Routes::Feeds, :rss_private +Invidious::Routing.get "/feed/playlist/:plid", Invidious::Routes::Feeds, :rss_playlist +Invidious::Routing.get "/feeds/videos.xml", Invidious::Routes::Feeds, :rss_videos + +# Support push notifications via PubSubHubbub +Invidious::Routing.get "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_get +Invidious::Routing.post "/feed/webhook/:token", Invidious::Routes::Feeds, :push_notifications_post + # Users post "/watch_ajax" do |env| @@ -1190,425 +1207,6 @@ post "/token_ajax" do |env| end end -# Feeds - -get "/feed/playlists" do |env| - env.redirect "/view_all_playlists" -end - -get "/feed/top" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - message = translate(locale, "The Top feed has been removed from Invidious.") - templated "message" -end - -get "/feed/popular" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - if CONFIG.popular_enabled - templated "popular" - else - message = translate(locale, "The Popular feed has been disabled by the administrator.") - templated "message" - end -end - -get "/feed/trending" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - trending_type = env.params.query["type"]? - trending_type ||= "Default" - - region = env.params.query["region"]? - region ||= "US" - - begin - trending, plid = fetch_trending(trending_type, region, locale) - rescue ex - next error_template(500, ex) - end - - templated "trending" -end - -get "/feed/subscriptions" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - sid = env.get? "sid" - referer = get_referer(env) - - if !user - next env.redirect referer - end - - user = user.as(User) - sid = sid.as(String) - token = user.token - - if user.preferences.unseen_only - env.set "show_watched", true - end - - # Refresh account - headers = HTTP::Headers.new - headers["Cookie"] = env.request.headers["Cookie"] - - if !user.password - user, sid = get_user(sid, headers, PG_DB) - end - - max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) - max_results ||= user.preferences.max_results - max_results ||= CONFIG.default_user_preferences.max_results - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) - - # "updated" here is used for delivering new notifications, so if - # we know a user has looked at their feed e.g. in the past 10 minutes, - # they've already seen a video posted 20 minutes ago, and don't need - # to be notified. - PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, - user.email) - user.notifications = [] of String - env.set "user", user - - templated "subscriptions" -end - -get "/feed/history" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - if !user - next env.redirect referer - end - - user = user.as(User) - - max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) - max_results ||= user.preferences.max_results - max_results ||= CONFIG.default_user_preferences.max_results - - if user.watched[(page - 1) * max_results]? - watched = user.watched.reverse[(page - 1) * max_results, max_results] - end - watched ||= [] of String - - templated "history" -end - -get "/feed/channel/:ucid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/atom+xml" - - ucid = env.params.url["ucid"] - - params = HTTP::Params.parse(env.params.query["params"]? || "") - - begin - channel = get_about_info(ucid, locale) - rescue ex : ChannelRedirect - next env.redirect env.request.resource.gsub(ucid, ex.channel_id) - rescue ex - next error_atom(500, ex) - end - - response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") - rss = XML.parse_html(response.body) - - videos = rss.xpath_nodes("//feed/entry").map do |entry| - video_id = entry.xpath_node("videoid").not_nil!.content - title = entry.xpath_node("title").not_nil!.content - - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - author = entry.xpath_node("author/name").not_nil!.content - ucid = entry.xpath_node("channelid").not_nil!.content - description_html = entry.xpath_node("group/description").not_nil!.to_s - views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 - - SearchVideo.new({ - title: title, - id: video_id, - author: author, - ucid: ucid, - published: published, - views: views, - description_html: description_html, - length_seconds: 0, - live_now: false, - premium: false, - premiere_timestamp: nil, - }) - end - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", - "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", - "xml:lang": "en-US") do - xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") - xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } - xml.element("yt:channelId") { xml.text channel.ucid } - xml.element("icon") { xml.text channel.author_thumbnail } - xml.element("title") { xml.text channel.author } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") - - xml.element("author") do - xml.element("name") { xml.text channel.author } - xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" } - end - - videos.each do |video| - video.to_xml(channel.auto_generated, params, xml) - end - end - end -end - -get "/feed/private" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/atom+xml" - - token = env.params.query["token"]? - - if !token - env.response.status_code = 403 - next - end - - user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User) - if !user - env.response.status_code = 403 - next - end - - max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) - max_results ||= user.preferences.max_results - max_results ||= CONFIG.default_user_preferences.max_results - - page = env.params.query["page"]?.try &.to_i? - page ||= 1 - - params = HTTP::Params.parse(env.params.query["params"]? || "") - - videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) - - XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", - "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", - "xml:lang": "en-US") do - xml.element("link", "type": "text/html", rel: "alternate", href: "#{HOST_URL}/feed/subscriptions") - xml.element("link", "type": "application/atom+xml", rel: "self", - href: "#{HOST_URL}#{env.request.resource}") - xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) } - - (notifications + videos).each do |video| - video.to_xml(locale, params, xml) - end - end - end -end - -get "/feed/playlist/:plid" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - env.response.content_type = "application/atom+xml" - - plid = env.params.url["plid"] - - params = HTTP::Params.parse(env.params.query["params"]? || "") - path = env.request.path - - if plid.starts_with? "IV" - if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) - videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) - - next XML.build(indent: " ", encoding: "UTF-8") do |xml| - xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", - "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", - "xml:lang": "en-US") do - xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") - xml.element("id") { xml.text "iv:playlist:#{plid}" } - xml.element("iv:playlistId") { xml.text plid } - xml.element("title") { xml.text playlist.title } - xml.element("link", rel: "alternate", href: "#{HOST_URL}/playlist?list=#{plid}") - - xml.element("author") do - xml.element("name") { xml.text playlist.author } - end - - videos.each do |video| - video.to_xml(false, xml) - end - end - end - else - env.response.status_code = 404 - next - end - end - - response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") - document = XML.parse(response.body) - - document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node| - node.attributes.each do |attribute| - case attribute.name - when "url", "href" - request_target = URI.parse(node[attribute.name]).request_target - query_string_opt = request_target.starts_with?("/watch?v=") ? "&#{params}" : "" - node[attribute.name] = "#{HOST_URL}#{request_target}#{query_string_opt}" - else nil # Skip - end - end - end - - document = document.to_xml(options: XML::SaveOptions::NO_DECL) - - document.scan(/(?[^<]+)<\/uri>/).each do |match| - content = "#{HOST_URL}#{URI.parse(match["url"]).request_target}" - document = document.gsub(match[0], "#{content}") - end - - document -end - -get "/feeds/videos.xml" do |env| - if ucid = env.params.query["channel_id"]? - env.redirect "/feed/channel/#{ucid}" - elsif user = env.params.query["user"]? - env.redirect "/feed/channel/#{user}" - elsif plid = env.params.query["playlist_id"]? - env.redirect "/feed/playlist/#{plid}" - end -end - -# Support push notifications via PubSubHubbub - -get "/feed/webhook/:token" do |env| - verify_token = env.params.url["token"] - - mode = env.params.query["hub.mode"]? - topic = env.params.query["hub.topic"]? - challenge = env.params.query["hub.challenge"]? - - if !mode || !topic || !challenge - env.response.status_code = 400 - next - else - mode = mode.not_nil! - topic = topic.not_nil! - challenge = challenge.not_nil! - end - - case verify_token - when .starts_with? "v1" - _, time, nonce, signature = verify_token.split(":") - data = "#{time}:#{nonce}" - when .starts_with? "v2" - time, signature = verify_token.split(":") - data = "#{time}" - else - env.response.status_code = 400 - next - end - - # The hub will sometimes check if we're still subscribed after delivery errors, - # so we reply with a 200 as long as the request hasn't expired - if Time.utc.to_unix - time.to_i > 432000 - env.response.status_code = 400 - next - end - - if OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, data) != signature - env.response.status_code = 400 - next - end - - if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? - PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) - elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? - PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) - else - env.response.status_code = 400 - next - end - - env.response.status_code = 200 - challenge -end - -post "/feed/webhook/:token" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - token = env.params.url["token"] - body = env.request.body.not_nil!.gets_to_end - signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") - - if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body) - LOGGER.error("/feed/webhook/#{token} : Invalid signature") - env.response.status_code = 200 - next - end - - spawn do - rss = XML.parse_html(body) - rss.xpath_nodes("//feed/entry").each do |entry| - id = entry.xpath_node("videoid").not_nil!.content - author = entry.xpath_node("author/name").not_nil!.content - published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) - updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) - - video = get_video(id, PG_DB, force_refresh: true) - - # Deliver notifications to `/api/v1/auth/notifications` - payload = { - "topic" => video.ucid, - "videoId" => video.id, - "published" => published.to_unix, - }.to_json - PG_DB.exec("NOTIFY notifications, E'#{payload}'") - - video = ChannelVideo.new({ - id: id, - title: video.title, - published: published, - updated: updated, - ucid: video.ucid, - author: author, - length_seconds: video.length_seconds, - live_now: video.live_now, - premiere_timestamp: video.premiere_timestamp, - views: video.views, - }) - - was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) - ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, - updated = $4, ucid = $5, author = $6, length_seconds = $7, - live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) - - PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), - feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert - end - end - - env.response.status_code = 200 - next -end - # Channels {"/channel/:ucid/live", "/user/:user/live", "/c/:user/live"}.each do |route| diff --git a/src/invidious/helpers/macros.cr b/src/invidious/helpers/macros.cr index 5d426a8b..75df1612 100644 --- a/src/invidious/helpers/macros.cr +++ b/src/invidious/helpers/macros.cr @@ -56,3 +56,12 @@ end macro rendered(filename) render "src/invidious/views/#{{{filename}}}.ecr" end + +# Similar to Kemals halt method but works in a +# method. +macro haltf(env, status_code = 200, response = "") + {{env}}.response.status_code = {{status_code}} + {{env}}.response.print {{response}} + {{env}}.response.close + return +end diff --git a/src/invidious/routes/feeds.cr b/src/invidious/routes/feeds.cr new file mode 100644 index 00000000..c88e96cf --- /dev/null +++ b/src/invidious/routes/feeds.cr @@ -0,0 +1,431 @@ +module Invidious::Routes::Feeds + def self.view_all_playlists_redirect(env) + env.redirect "/feed/playlists" + end + + def self.playlists(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + return env.redirect "/" if user.nil? + + user = user.as(User) + + items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_created.map! do |item| + item.author = "" + item + end + + items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) + items_saved.map! do |item| + item.author = "" + item + end + + templated "feeds/playlists" + end + + def self.popular(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + if CONFIG.popular_enabled + templated "feeds/popular" + else + message = translate(locale, "The Popular feed has been disabled by the administrator.") + templated "message" + end + end + + def self.trending(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + trending_type = env.params.query["type"]? + trending_type ||= "Default" + + region = env.params.query["region"]? + region ||= "US" + + begin + trending, plid = fetch_trending(trending_type, region, locale) + rescue ex + return error_template(500, ex) + end + + templated "feeds/trending" + end + + def self.subscriptions(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + sid = env.get? "sid" + referer = get_referer(env) + + if !user + return env.redirect referer + end + + user = user.as(User) + sid = sid.as(String) + token = user.token + + if user.preferences.unseen_only + env.set "show_watched", true + end + + # Refresh account + headers = HTTP::Headers.new + headers["Cookie"] = env.request.headers["Cookie"] + + if !user.password + user, sid = get_user(sid, headers, PG_DB) + end + + max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + + # "updated" here is used for delivering new notifications, so if + # we know a user has looked at their feed e.g. in the past 10 minutes, + # they've already seen a video posted 20 minutes ago, and don't need + # to be notified. + PG_DB.exec("UPDATE users SET notifications = $1, updated = $2 WHERE email = $3", [] of String, Time.utc, + user.email) + user.notifications = [] of String + env.set "user", user + + templated "feeds/subscriptions" + end + + def self.history(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.get? "user" + referer = get_referer(env) + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + if !user + return env.redirect referer + end + + user = user.as(User) + + max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + if user.watched[(page - 1) * max_results]? + watched = user.watched.reverse[(page - 1) * max_results, max_results] + end + watched ||= [] of String + + templated "feeds/history" + end + + # RSS feeds + + def self.rss_channel(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.headers["Content-Type"] = "application/atom+xml" + env.response.content_type = "application/atom+xml" + + ucid = env.params.url["ucid"] + + params = HTTP::Params.parse(env.params.query["params"]? || "") + + begin + channel = get_about_info(ucid, locale) + rescue ex : ChannelRedirect + return env.redirect env.request.resource.gsub(ucid, ex.channel_id) + rescue ex + return error_atom(500, ex) + end + + response = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{channel.ucid}") + rss = XML.parse_html(response.body) + + videos = rss.xpath_nodes("//feed/entry").map do |entry| + video_id = entry.xpath_node("videoid").not_nil!.content + title = entry.xpath_node("title").not_nil!.content + + published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) + + author = entry.xpath_node("author/name").not_nil!.content + ucid = entry.xpath_node("channelid").not_nil!.content + description_html = entry.xpath_node("group/description").not_nil!.to_s + views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 + + SearchVideo.new({ + title: title, + id: video_id, + author: author, + ucid: ucid, + published: published, + views: views, + description_html: description_html, + length_seconds: 0, + live_now: false, + paid: false, + premium: false, + premiere_timestamp: nil, + }) + end + + XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", + "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", + "xml:lang": "en-US") do + xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") + xml.element("id") { xml.text "yt:channel:#{channel.ucid}" } + xml.element("yt:channelId") { xml.text channel.ucid } + xml.element("icon") { xml.text channel.author_thumbnail } + xml.element("title") { xml.text channel.author } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/channel/#{channel.ucid}") + + xml.element("author") do + xml.element("name") { xml.text channel.author } + xml.element("uri") { xml.text "#{HOST_URL}/channel/#{channel.ucid}" } + end + + videos.each do |video| + video.to_xml(channel.auto_generated, params, xml) + end + end + end + end + + def self.rss_private(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.headers["Content-Type"] = "application/atom+xml" + env.response.content_type = "application/atom+xml" + + token = env.params.query["token"]? + + if !token + haltf env, status_code: 403 + end + + user = PG_DB.query_one?("SELECT * FROM users WHERE token = $1", token.strip, as: User) + if !user + haltf env, status_code: 403 + end + + max_results = env.params.query["max_results"]?.try &.to_i?.try &.clamp(0, MAX_ITEMS_PER_PAGE) + max_results ||= user.preferences.max_results + max_results ||= CONFIG.default_user_preferences.max_results + + page = env.params.query["page"]?.try &.to_i? + page ||= 1 + + params = HTTP::Params.parse(env.params.query["params"]? || "") + + videos, notifications = get_subscription_feed(PG_DB, user, max_results, page) + + XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", + "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", + "xml:lang": "en-US") do + xml.element("link", "type": "text/html", rel: "alternate", href: "#{HOST_URL}/feed/subscriptions") + xml.element("link", "type": "application/atom+xml", rel: "self", + href: "#{HOST_URL}#{env.request.resource}") + xml.element("title") { xml.text translate(locale, "Invidious Private Feed for `x`", user.email) } + + (notifications + videos).each do |video| + video.to_xml(locale, params, xml) + end + end + end + end + + def self.rss_playlist(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + env.response.headers["Content-Type"] = "application/atom+xml" + env.response.content_type = "application/atom+xml" + + plid = env.params.url["plid"] + + params = HTTP::Params.parse(env.params.query["params"]? || "") + path = env.request.path + + if plid.starts_with? "IV" + if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) + videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) + + return XML.build(indent: " ", encoding: "UTF-8") do |xml| + xml.element("feed", "xmlns:yt": "http://www.youtube.com/xml/schemas/2015", + "xmlns:media": "http://search.yahoo.com/mrss/", xmlns: "http://www.w3.org/2005/Atom", + "xml:lang": "en-US") do + xml.element("link", rel: "self", href: "#{HOST_URL}#{env.request.resource}") + xml.element("id") { xml.text "iv:playlist:#{plid}" } + xml.element("iv:playlistId") { xml.text plid } + xml.element("title") { xml.text playlist.title } + xml.element("link", rel: "alternate", href: "#{HOST_URL}/playlist?list=#{plid}") + + xml.element("author") do + xml.element("name") { xml.text playlist.author } + end + + videos.each do |video| + video.to_xml(false, xml) + end + end + end + else + haltf env, status_code: 404 + end + end + + response = YT_POOL.client &.get("/feeds/videos.xml?playlist_id=#{plid}") + document = XML.parse(response.body) + + document.xpath_nodes(%q(//*[@href]|//*[@url])).each do |node| + node.attributes.each do |attribute| + case attribute.name + when "url", "href" + request_target = URI.parse(node[attribute.name]).request_target + query_string_opt = request_target.starts_with?("/watch?v=") ? "&#{params}" : "" + node[attribute.name] = "#{HOST_URL}#{request_target}#{query_string_opt}" + else nil # Skip + end + end + end + + document = document.to_xml(options: XML::SaveOptions::NO_DECL) + + document.scan(/(?[^<]+)<\/uri>/).each do |match| + content = "#{HOST_URL}#{URI.parse(match["url"]).request_target}" + document = document.gsub(match[0], "#{content}") + end + document + end + + def self.rss_videos(env) + if ucid = env.params.query["channel_id"]? + env.redirect "/feed/channel/#{ucid}" + elsif user = env.params.query["user"]? + env.redirect "/feed/channel/#{user}" + elsif plid = env.params.query["playlist_id"]? + env.redirect "/feed/playlist/#{plid}" + end + end + + # Push notifications via PubSub + + def self.push_notifications_get(env) + verify_token = env.params.url["token"] + + mode = env.params.query["hub.mode"]? + topic = env.params.query["hub.topic"]? + challenge = env.params.query["hub.challenge"]? + + if !mode || !topic || !challenge + haltf env, status_code: 400 + else + mode = mode.not_nil! + topic = topic.not_nil! + challenge = challenge.not_nil! + end + + case verify_token + when .starts_with? "v1" + _, time, nonce, signature = verify_token.split(":") + data = "#{time}:#{nonce}" + when .starts_with? "v2" + time, signature = verify_token.split(":") + data = "#{time}" + else + haltf env, status_code: 400 + end + + # The hub will sometimes check if we're still subscribed after delivery errors, + # so we reply with a 200 as long as the request hasn't expired + if Time.utc.to_unix - time.to_i > 432000 + haltf env, status_code: 400 + end + + if OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, data) != signature + haltf env, status_code: 400 + end + + if ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"]? + PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + elsif plid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["playlist_id"]? + PG_DB.exec("UPDATE playlists SET subscribed = $1 WHERE id = $2", Time.utc, ucid) + else + haltf env, status_code: 400 + end + + env.response.status_code = 200 + challenge + end + + def self.push_notifications_post(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + token = env.params.url["token"] + body = env.request.body.not_nil!.gets_to_end + signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") + + if signature != OpenSSL::HMAC.hexdigest(:sha1, HMAC_KEY, body) + LOGGER.error("/feed/webhook/#{token} : Invalid signature") + haltf env, status_code: 200 + end + + spawn do + rss = XML.parse_html(body) + rss.xpath_nodes("//feed/entry").each do |entry| + id = entry.xpath_node("videoid").not_nil!.content + author = entry.xpath_node("author/name").not_nil!.content + published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content) + updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) + + video = get_video(id, PG_DB, force_refresh: true) + + # Deliver notifications to `/api/v1/auth/notifications` + payload = { + "topic" => video.ucid, + "videoId" => video.id, + "published" => published.to_unix, + }.to_json + PG_DB.exec("NOTIFY notifications, E'#{payload}'") + + video = ChannelVideo.new({ + id: id, + title: video.title, + published: published, + updated: updated, + ucid: video.ucid, + author: author, + length_seconds: video.length_seconds, + live_now: video.live_now, + premiere_timestamp: video.premiere_timestamp, + views: video.views, + }) + + was_insert = PG_DB.query_one("INSERT INTO channel_videos VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, + updated = $4, ucid = $5, author = $6, length_seconds = $7, + live_now = $8, premiere_timestamp = $9, views = $10 returning (xmax=0) as was_insert", *video.to_tuple, as: Bool) + + PG_DB.exec("UPDATE users SET notifications = array_append(notifications, $1), + feed_needs_update = true WHERE $2 = ANY(subscriptions)", video.id, video.ucid) if was_insert + end + end + + env.response.status_code = 200 + end +end diff --git a/src/invidious/routes/misc.cr b/src/invidious/routes/misc.cr index fa548f53..82c40a95 100644 --- a/src/invidious/routes/misc.cr +++ b/src/invidious/routes/misc.cr @@ -17,7 +17,7 @@ module Invidious::Routes::Misc end when "Playlists" if user - env.redirect "/view_all_playlists" + env.redirect "/feed/playlists" else env.redirect "/feed/popular" end diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index a2166bdd..05a198d8 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -1,29 +1,4 @@ module Invidious::Routes::Playlists - def self.index(env) - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.get? "user" - referer = get_referer(env) - - return env.redirect "/" if user.nil? - - user = user.as(User) - - items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_created.map! do |item| - item.author = "" - item - end - - items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) - items_saved.map! do |item| - item.author = "" - item - end - - templated "view_all_playlists" - end - def self.new(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? @@ -148,7 +123,7 @@ module Invidious::Routes::Playlists PG_DB.exec("DELETE FROM playlist_videos * WHERE plid = $1", plid) PG_DB.exec("DELETE FROM playlists * WHERE id = $1", plid) - env.redirect "/view_all_playlists" + env.redirect "/feed/playlists" end def self.edit(env) diff --git a/src/invidious/views/history.ecr b/src/invidious/views/feeds/history.ecr similarity index 100% rename from src/invidious/views/history.ecr rename to src/invidious/views/feeds/history.ecr diff --git a/src/invidious/views/view_all_playlists.ecr b/src/invidious/views/feeds/playlists.ecr similarity index 100% rename from src/invidious/views/view_all_playlists.ecr rename to src/invidious/views/feeds/playlists.ecr diff --git a/src/invidious/views/popular.ecr b/src/invidious/views/feeds/popular.ecr similarity index 100% rename from src/invidious/views/popular.ecr rename to src/invidious/views/feeds/popular.ecr diff --git a/src/invidious/views/subscriptions.ecr b/src/invidious/views/feeds/subscriptions.ecr similarity index 100% rename from src/invidious/views/subscriptions.ecr rename to src/invidious/views/feeds/subscriptions.ecr diff --git a/src/invidious/views/trending.ecr b/src/invidious/views/feeds/trending.ecr similarity index 100% rename from src/invidious/views/trending.ecr rename to src/invidious/views/feeds/trending.ecr diff --git a/src/invidious/views/playlist.ecr b/src/invidious/views/playlist.ecr index b1fee211..12f93a72 100644 --- a/src/invidious/views/playlist.ecr +++ b/src/invidious/views/playlist.ecr @@ -12,7 +12,7 @@ <% if playlist.is_a? InvidiousPlaylist %> <% if playlist.author == user.try &.email %> - <%= author %> | + <%= author %> | <% else %> <%= author %> | <% end %> diff --git a/src/invidious/views/preferences.ecr b/src/invidious/views/preferences.ecr index d98c3bb5..be021c59 100644 --- a/src/invidious/views/preferences.ecr +++ b/src/invidious/views/preferences.ecr @@ -312,7 +312,7 @@
- <%= translate(locale, "View all playlists") %> + <%= translate(locale, "View all playlists") %>