From fd26f9f34e6ee9383ae9b9c94f8287c9100c7726 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Fri, 22 Mar 2019 12:24:47 -0500 Subject: [PATCH] Add support for premieres to search and feed --- config/migrate-scripts/migrate-db-3864cbf.sh | 3 + config/migrate-scripts/migrate-db-6e51189.sh | 2 +- ...te-db-3646395.sh => migrate-db-88b7097.sh} | 0 config/sql/channel_videos.sql | 2 + src/invidious.cr | 12 ++-- src/invidious/channels.cr | 56 ++++++++++++++----- src/invidious/helpers/helpers.cr | 8 ++- src/invidious/search.cr | 25 +++++---- src/invidious/videos.cr | 8 ++- src/invidious/views/components/item.ecr | 12 ++-- 10 files changed, 89 insertions(+), 39 deletions(-) create mode 100755 config/migrate-scripts/migrate-db-3864cbf.sh rename config/migrate-scripts/{migrate-db-3646395.sh => migrate-db-88b7097.sh} (100%) diff --git a/config/migrate-scripts/migrate-db-3864cbf.sh b/config/migrate-scripts/migrate-db-3864cbf.sh new file mode 100755 index 00000000..c63d37f0 --- /dev/null +++ b/config/migrate-scripts/migrate-db-3864cbf.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +psql invidious -c "ALTER TABLE channel_videos ADD COLUMN premiere_timestamp timestamptz;" diff --git a/config/migrate-scripts/migrate-db-6e51189.sh b/config/migrate-scripts/migrate-db-6e51189.sh index 80f222bb..adadc109 100755 --- a/config/migrate-scripts/migrate-db-6e51189.sh +++ b/config/migrate-scripts/migrate-db-6e51189.sh @@ -1,4 +1,4 @@ #!/bin/sh psql invidious -c "ALTER TABLE channel_videos ADD COLUMN live_now bool;" -psql invidious -c "UPDATE channel_videos SET live_now = false;" \ No newline at end of file +psql invidious -c "UPDATE channel_videos SET live_now = false;" diff --git a/config/migrate-scripts/migrate-db-3646395.sh b/config/migrate-scripts/migrate-db-88b7097.sh similarity index 100% rename from config/migrate-scripts/migrate-db-3646395.sh rename to config/migrate-scripts/migrate-db-88b7097.sh diff --git a/config/sql/channel_videos.sql b/config/sql/channel_videos.sql index 279fad29..2fae636e 100644 --- a/config/sql/channel_videos.sql +++ b/config/sql/channel_videos.sql @@ -11,6 +11,8 @@ CREATE TABLE public.channel_videos ucid text, author text, length_seconds integer, + live_now boolean, + premiere_timestamp timestamp with time zone, CONSTRAINT channel_videos_id_key UNIQUE (id) ); diff --git a/src/invidious.cr b/src/invidious.cr index 0ff28e11..068ad201 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -2104,7 +2104,8 @@ get "/feed/channel/:ucid" do |env| length_seconds: 0, live_now: false, paid: false, - premium: false + premium: false, + premiere_timestamp: nil ) end @@ -2372,7 +2373,7 @@ post "/feed/webhook/:token" do |env| updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content) video = get_video(id, PG_DB, proxies, region: nil) - video = ChannelVideo.new(id, video.title, published, updated, video.ucid, video.author, video.length_seconds, video.live_now) + video = ChannelVideo.new(id, video.title, published, updated, video.ucid, video.author, video.length_seconds, video.live_now, video.premiere_timestamp) PG_DB.exec("UPDATE users SET notifications = notifications || $1 \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, video.ucid) @@ -2382,7 +2383,8 @@ post "/feed/webhook/:token" do |env| PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array) + updated = $4, ucid = $5, author = $6, length_seconds = $7, \ + live_now = $8, premiere_timestamp = $9", video_array) end end @@ -2901,8 +2903,8 @@ get "/api/v1/videos/:id" do |env| json.field "liveNow", video.live_now json.field "isUpcoming", video.is_upcoming - if video.is_upcoming - json.field "premiereTimestamp", video.premiere_timestamp + if video.premiere_timestamp + json.field "premiereTimestamp", video.premiere_timestamp.not_nil!.to_unix end if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]? diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index 57820c49..126cc2b8 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -10,14 +10,15 @@ end class ChannelVideo add_mapping({ - id: String, - title: String, - published: Time, - updated: Time, - ucid: String, - author: String, - length_seconds: {type: Int32, default: 0}, - live_now: {type: Bool, default: false}, + id: String, + title: String, + published: Time, + updated: Time, + ucid: String, + author: String, + length_seconds: {type: Int32, default: 0}, + live_now: {type: Bool, default: false}, + premiere_timestamp: {type: Time?, default: nil}, }) end @@ -126,7 +127,19 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) live_now = channel_video.try &.live_now live_now ||= false - video = ChannelVideo.new(video_id, title, published, Time.now, ucid, author, length_seconds, live_now) + premiere_timestamp = channel_video.try &.premiere_timestamp + + video = ChannelVideo.new( + video_id, + title, + published, + Time.now, + ucid, + author, + length_seconds, + live_now, + premiere_timestamp + ) db.exec("UPDATE users SET notifications = notifications || $1 \ WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid) @@ -134,9 +147,12 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) video_array = video.to_a args = arg_array(video_array) + # We don't include the 'premire_timestamp' here because channel pages don't include them, + # meaning the above timestamp is always null db.exec("INSERT INTO channel_videos VALUES (#{args}) \ ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ - updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array) + updated = $4, ucid = $5, author = $6, length_seconds = $7, \ + live_now = $8", video_array) end else page = 1 @@ -163,7 +179,17 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) end count = nodeset.size - videos = videos.map { |video| ChannelVideo.new(video.id, video.title, video.published, Time.now, video.ucid, video.author, video.length_seconds, video.live_now) } + videos = videos.map { |video| ChannelVideo.new( + video.id, + video.title, + video.published, + Time.now, + video.ucid, + video.author, + video.length_seconds, + video.live_now, + video.premiere_timestamp + ) } videos.each do |video| ids << video.id @@ -176,8 +202,12 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) video_array = video.to_a args = arg_array(video_array) - db.exec("INSERT INTO channel_videos VALUES (#{args}) ON CONFLICT (id) DO UPDATE SET title = $2, \ - published = $3, updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array) + # We don't include the 'premire_timestamp' here because channel pages don't include them, + # meaning the above timestamp is always null + db.exec("INSERT INTO channel_videos VALUES (#{args}) \ + ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \ + updated = $4, ucid = $5, author = $6, length_seconds = $7, \ + live_now = $8", video_array) end end diff --git a/src/invidious/helpers/helpers.cr b/src/invidious/helpers/helpers.cr index 271aa0c4..4e827bd8 100644 --- a/src/invidious/helpers/helpers.cr +++ b/src/invidious/helpers/helpers.cr @@ -325,6 +325,11 @@ def extract_items(nodeset, ucid = nil, author_name = nil) paid = true end + premiere_timestamp = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/span[@class="localized-date"])).try &.["data-timestamp"]?.try &.to_i64 + if premiere_timestamp + premiere_timestamp = Time.unix(premiere_timestamp) + end + items << SearchVideo.new( title: title, id: id, @@ -337,7 +342,8 @@ def extract_items(nodeset, ucid = nil, author_name = nil) length_seconds: length_seconds, live_now: live_now, paid: paid, - premium: premium + premium: premium, + premiere_timestamp: premiere_timestamp ) end end diff --git a/src/invidious/search.cr b/src/invidious/search.cr index 63cce4de..6805f119 100644 --- a/src/invidious/search.cr +++ b/src/invidious/search.cr @@ -1,17 +1,18 @@ class SearchVideo add_mapping({ - title: String, - id: String, - author: String, - ucid: String, - published: Time, - views: Int64, - description: String, - description_html: String, - length_seconds: Int32, - live_now: Bool, - paid: Bool, - premium: Bool, + title: String, + id: String, + author: String, + ucid: String, + published: Time, + views: Int64, + description: String, + description_html: String, + length_seconds: Int32, + live_now: Bool, + paid: Bool, + premium: Bool, + premiere_timestamp: Time?, }) end diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 9ba9c2a1..f89dabd7 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -298,11 +298,13 @@ class Video .try &.["offlineSlate"]? .try &.["liveStreamOfflineSlateRenderer"]? .try &.["scheduledStartTime"].as_s.to_i64 + end - return premiere_timestamp - else - return nil + if premiere_timestamp + premiere_timestamp = Time.unix(premiere_timestamp) end + + return premiere_timestamp end def keywords diff --git a/src/invidious/views/components/item.ecr b/src/invidious/views/components/item.ecr index cd457324..8226dc6f 100644 --- a/src/invidious/views/components/item.ecr +++ b/src/invidious/views/components/item.ecr @@ -67,8 +67,10 @@

<%= item.author %>

- - <% if Time.now - item.published > 1.minute %> + + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %> +
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %>
+ <% elsif Time.now - item.published > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% end %> <% else %> @@ -102,8 +104,10 @@

<%= item.author %>

- - <% if Time.now - item.published > 1.minute %> + + <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %> +
<%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %>
+ <% elsif Time.now - item.published > 1.minute %>
<%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %>
<% end %> <% end %>