From 19ed5bf993a070534e1c0745de16f03c7409b738 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 22 Apr 2019 15:39:57 -0500 Subject: [PATCH] Add support for 'user' URLs in NewPipe import --- src/invidious.cr | 18 ++++++++++++++++-- src/invidious/channels.cr | 3 +-- src/invidious/search.cr | 6 ++++++ 3 files changed, 23 insertions(+), 4 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index 4a15602e..0cabd507 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -1842,8 +1842,22 @@ post "/data_control" do |env| PG_DB.exec("UPDATE users SET subscriptions = $1 WHERE email = $2", user.subscriptions, user.email) when "import_newpipe_subscriptions" body = JSON.parse(body) - user.subscriptions += body["subscriptions"].as_a.map do |channel| - channel["url"].as_s.match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0] + user.subscriptions += body["subscriptions"].as_a.compact_map do |channel| + if match = channel["url"].as_s.match(/\/channel\/(?UC[a-zA-Z0-9_-]{22})/) + next match["channel"] + elsif match = channel["url"].as_s.match(/\/user\/(?.+)/) + client = make_client(YT_URL) + response = client.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US") + document = XML.parse_html(response.body) + canonical = document.xpath_node(%q(//link[@rel="canonical"])) + + if canonical + ucid = canonical["href"].split("/")[-1] + next ucid + end + end + + nil end user.subscriptions.uniq! diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 060d5c2e..96544b49 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -51,8 +51,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma final = [] of String channels.size.times do - ucid = finished_channel.receive - if ucid + if ucid = finished_channel.receive final << ucid end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index c3c48af3..58ccb164 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -63,6 +63,12 @@ def channel_search(query, page, channel) canonical = document.xpath_node(%q(//link[@rel="canonical"])) end + if !canonical + response = client.get("/user/#{channel}?disable_polymer=1&hl=en&gl=US") + document = XML.parse_html(response.body) + canonical = document.xpath_node(%q(//link[@rel="canonical"])) + end + if !canonical return 0, [] of SearchItem end