From 7425700009b2e127775ba33b9b9cf661867ecc2e Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sun, 3 Mar 2019 20:40:24 -0600 Subject: [PATCH] Update pubsub to support lease_seconds --- config/migrate-scripts/migrate-db-8e884fe1.sh | 4 ++++ config/sql/channels.sql | 1 + src/invidious.cr | 9 +++++---- src/invidious/channels.cr | 16 ++++++++-------- src/invidious/jobs.cr | 2 +- 5 files changed, 19 insertions(+), 13 deletions(-) create mode 100755 config/migrate-scripts/migrate-db-8e884fe1.sh diff --git a/config/migrate-scripts/migrate-db-8e884fe1.sh b/config/migrate-scripts/migrate-db-8e884fe1.sh new file mode 100755 index 00000000..ca137a5f --- /dev/null +++ b/config/migrate-scripts/migrate-db-8e884fe1.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +psql invidious -c "ALTER TABLE channels DROP COLUMN subscribed" +psql invidious -c "ALTER TABLE channels ADD COLUMN subscribed timestamptz" diff --git a/config/sql/channels.sql b/config/sql/channels.sql index cecd04ed..b5a29b8f 100644 --- a/config/sql/channels.sql +++ b/config/sql/channels.sql @@ -8,6 +8,7 @@ CREATE TABLE public.channels author text, updated timestamp with time zone, deleted boolean, + subscribed timestamp with time zone, CONSTRAINT channels_id_key UNIQUE (id) ); diff --git a/src/invidious.cr b/src/invidious.cr index 8a52253a..45abecc9 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2319,12 +2319,13 @@ end # Add support for subscribing to channels via PubSubHubbub -get "/feed/webhook" do |env| +get "/feed/webhook/:token" do |env| + verify_token = env.params.url["token"] + mode = env.params.query["hub.mode"] topic = env.params.query["hub.topic"] challenge = env.params.query["hub.challenge"] lease_seconds = env.params.query["hub.lease_seconds"] - verify_token = env.params.query["hub.verify_token"] time, signature = verify_token.split(":") @@ -2337,12 +2338,12 @@ get "/feed/webhook" do |env| end ucid = HTTP::Params.parse(URI.parse(topic).query.not_nil!)["channel_id"] - PG_DB.exec("UPDATE channels SET subscribed = true WHERE id = $1", ucid) + PG_DB.exec("UPDATE channels SET subscribed = $1 WHERE id = $2", Time.now, ucid) halt env, status_code: 200, response: challenge end -post "/feed/webhook" do |env| +post "/feed/webhook/:token" do |env| body = env.request.body.not_nil!.gets_to_end signature = env.request.headers["X-Hub-Signature"].lchop("sha1=") diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 097aa764..bb548045 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -4,7 +4,7 @@ class InvidiousChannel author: String, updated: Time, deleted: Bool, - subscribed: {type: Bool, default: false}, + subscribed: Time?, }) end @@ -186,7 +186,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) db.exec("DELETE FROM channel_videos * WHERE NOT id = ANY ('{#{ids.map { |id| %("#{id}") }.join(",")}}') AND ucid = $1", ucid) end - channel = InvidiousChannel.new(ucid, author, Time.now, false, false) + channel = InvidiousChannel.new(ucid, author, Time.now, false, nil) return channel end @@ -198,12 +198,12 @@ def subscribe_pubsub(ucid, key, config) host_url = make_host_url(Kemal.config.ssl || config.https_only, config.domain) body = { - "hub.callback" => "#{host_url}/feed/webhook", - "hub.topic" => "https://www.youtube.com/feeds/videos.xml?channel_id=#{ucid}", - "hub.verify" => "async", - "hub.mode" => "subscribe", - "hub.verify_token" => "#{time}:#{OpenSSL::HMAC.hexdigest(:sha1, key, time)}", - "hub.secret" => key.to_s, + "hub.callback" => "#{host_url}/feed/webhook/#{time}:#{OpenSSL::HMAC.hexdigest(:sha1, key, time)}", + "hub.topic" => "https://www.youtube.com/feeds/videos.xml?channel_id=#{ucid}", + "hub.verify" => "async", + "hub.mode" => "subscribe", + "hub.lease_seconds" => "432000", + "hub.secret" => key.to_s, } return client.post("/subscribe", form: body) diff --git a/src/invidious/jobs.cr b/src/invidious/jobs.cr index a5acd4cf..49745aba 100644 --- a/src/invidious/jobs.cr +++ b/src/invidious/jobs.cr @@ -157,7 +157,7 @@ def subscribe_to_feeds(db, logger, key, config) if config.use_pubsub_feeds spawn do loop do - db.query_all("SELECT id FROM channels WHERE subscribed = false") do |rs| + db.query_all("SELECT id FROM channels WHERE CURRENT_TIMESTAMP - subscribed > '4 days'") do |rs| ucid = rs.read(String) response = subscribe_pubsub(ucid, key, config)