Add support for premieres to search and feed

config/migrate-scripts/migrate-db-3864cbf.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+psql invidious -c "ALTER TABLE channel_videos ADD COLUMN premiere_timestamp timestamptz;"

config/sql/channel_videos.sql

@@ -11,6 +11,8 @@ CREATE TABLE public.channel_videos
   ucid text,
   author text,
   length_seconds integer,
+  live_now boolean,
+  premiere_timestamp timestamp with time zone,
   CONSTRAINT channel_videos_id_key UNIQUE (id)
 );
 

src/invidious.cr

@@ -2104,7 +2104,8 @@ get "/feed/channel/:ucid" do |env|
       length_seconds: 0,
       live_now: false,
       paid: false,
-      premium: false
+      premium: false,
+      premiere_timestamp: nil
     )
   end
 
@@ -2372,7 +2373,7 @@ post "/feed/webhook/:token" do |env|
       updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
 
       video = get_video(id, PG_DB, proxies, region: nil)
-      video = ChannelVideo.new(id, video.title, published, updated, video.ucid, video.author, video.length_seconds, video.live_now)
+      video = ChannelVideo.new(id, video.title, published, updated, video.ucid, video.author, video.length_seconds, video.live_now, video.premiere_timestamp)
 
       PG_DB.exec("UPDATE users SET notifications = notifications || $1 \
       WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, video.ucid)
@@ -2382,7 +2383,8 @@ post "/feed/webhook/:token" do |env|
 
       PG_DB.exec("INSERT INTO channel_videos VALUES (#{args}) \
       ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
-      updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array)
+      updated = $4, ucid = $5, author = $6, length_seconds = $7, \
+      live_now = $8, premiere_timestamp = $9", video_array)
     end
   end
 
@@ -2901,8 +2903,8 @@ get "/api/v1/videos/:id" do |env|
       json.field "liveNow", video.live_now
       json.field "isUpcoming", video.is_upcoming
 
-      if video.is_upcoming
+      if video.premiere_timestamp
-        json.field "premiereTimestamp", video.premiere_timestamp
+        json.field "premiereTimestamp", video.premiere_timestamp.not_nil!.to_unix
       end
 
       if video.player_response["streamingData"]?.try &.["hlsManifestUrl"]?

src/invidious/channels.cr

@@ -10,14 +10,15 @@ end
 
 class ChannelVideo
   add_mapping({
-    id:             String,
+    id:                 String,
-    title:          String,
+    title:              String,
-    published:      Time,
+    published:          Time,
-    updated:        Time,
+    updated:            Time,
-    ucid:           String,
+    ucid:               String,
-    author:         String,
+    author:             String,
-    length_seconds: {type: Int32, default: 0},
+    length_seconds:     {type: Int32, default: 0},
-    live_now:       {type: Bool, default: false},
+    live_now:           {type: Bool, default: false},
+    premiere_timestamp: {type: Time?, default: nil},
   })
 end
 
@@ -126,7 +127,19 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
       live_now = channel_video.try &.live_now
       live_now ||= false
 
-      video = ChannelVideo.new(video_id, title, published, Time.now, ucid, author, length_seconds, live_now)
+      premiere_timestamp = channel_video.try &.premiere_timestamp
+
+      video = ChannelVideo.new(
+        video_id,
+        title,
+        published,
+        Time.now,
+        ucid,
+        author,
+        length_seconds,
+        live_now,
+        premiere_timestamp
+      )
 
       db.exec("UPDATE users SET notifications = notifications || $1 \
         WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid)
@@ -134,9 +147,12 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
       video_array = video.to_a
       args = arg_array(video_array)
 
+      # We don't include the 'premire_timestamp' here because channel pages don't include them,
+      # meaning the above timestamp is always null
       db.exec("INSERT INTO channel_videos VALUES (#{args}) \
       ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
-      updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array)
+      updated = $4, ucid = $5, author = $6, length_seconds = $7, \
+      live_now = $8", video_array)
     end
   else
     page = 1
@@ -163,7 +179,17 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
       end
 
       count = nodeset.size
-      videos = videos.map { |video| ChannelVideo.new(video.id, video.title, video.published, Time.now, video.ucid, video.author, video.length_seconds, video.live_now) }
+      videos = videos.map { |video| ChannelVideo.new(
+        video.id,
+        video.title,
+        video.published,
+        Time.now,
+        video.ucid,
+        video.author,
+        video.length_seconds,
+        video.live_now,
+        video.premiere_timestamp
+      ) }
 
       videos.each do |video|
         ids << video.id
@@ -176,8 +202,12 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
           video_array = video.to_a
           args = arg_array(video_array)
 
-          db.exec("INSERT INTO channel_videos VALUES (#{args}) ON CONFLICT (id) DO UPDATE SET title = $2, \
+          # We don't include the 'premire_timestamp' here because channel pages don't include them,
-          published = $3, updated = $4, ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array)
+          # meaning the above timestamp is always null
+          db.exec("INSERT INTO channel_videos VALUES (#{args}) \
+          ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
+          updated = $4, ucid = $5, author = $6, length_seconds = $7, \
+          live_now = $8", video_array)
         end
       end
 

src/invidious/helpers/helpers.cr

@@ -325,6 +325,11 @@ def extract_items(nodeset, ucid = nil, author_name = nil)
         paid = true
       end
 
+      premiere_timestamp = node.xpath_node(%q(.//ul[@class="yt-lockup-meta-info"]/li/span[@class="localized-date"])).try &.["data-timestamp"]?.try &.to_i64
+      if premiere_timestamp
+        premiere_timestamp = Time.unix(premiere_timestamp)
+      end
+
       items << SearchVideo.new(
         title: title,
         id: id,
@@ -337,7 +342,8 @@ def extract_items(nodeset, ucid = nil, author_name = nil)
         length_seconds: length_seconds,
         live_now: live_now,
         paid: paid,
-        premium: premium
+        premium: premium,
+        premiere_timestamp: premiere_timestamp
       )
     end
   end

src/invidious/search.cr

@@ -1,17 +1,18 @@
 class SearchVideo
   add_mapping({
-    title:            String,
+    title:              String,
-    id:               String,
+    id:                 String,
-    author:           String,
+    author:             String,
-    ucid:             String,
+    ucid:               String,
-    published:        Time,
+    published:          Time,
-    views:            Int64,
+    views:              Int64,
-    description:      String,
+    description:        String,
-    description_html: String,
+    description_html:   String,
-    length_seconds:   Int32,
+    length_seconds:     Int32,
-    live_now:         Bool,
+    live_now:           Bool,
-    paid:             Bool,
+    paid:               Bool,
-    premium:          Bool,
+    premium:            Bool,
+    premiere_timestamp: Time?,
   })
 end
 

src/invidious/videos.cr

@@ -298,11 +298,13 @@ class Video
             .try &.["offlineSlate"]?
               .try &.["liveStreamOfflineSlateRenderer"]?
                 .try &.["scheduledStartTime"].as_s.to_i64
+    end
 
-      return premiere_timestamp
+    if premiere_timestamp
-    else
+      premiere_timestamp = Time.unix(premiere_timestamp)
-      return nil
     end
+
+    return premiere_timestamp
   end
 
   def keywords

src/invidious/views/components/item.ecr

@@ -68,7 +68,9 @@
             <b><a style="width:100%;" href="/channel/<%= item.ucid %>"><%= item.author %></a></b>
         </p>
 
-        <% if Time.now - item.published > 1.minute %>
+        <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %>
+        <h5><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %></h5>
+        <% elsif Time.now - item.published > 1.minute %>
         <h5><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></h5>
         <% end %>
     <% else %>
@@ -103,7 +105,9 @@
             <b><a style="width:100%;" href="/channel/<%= item.ucid %>"><%= item.author %></a></b>
         </p>
 
-        <% if Time.now - item.published > 1.minute %>
+        <% if item.responds_to?(:premiere_timestamp) && item.premiere_timestamp && item.premiere_timestamp.not_nil! > Time.now %>
+        <h5><%= translate(locale, "Premieres in `x`", recode_date((item.premiere_timestamp.as(Time) - Time.now).ago, locale)) %></h5>
+        <% elsif Time.now - item.published > 1.minute %>
         <h5><%= translate(locale, "Shared `x` ago", recode_date(item.published, locale)) %></h5>
         <% end %>
     <% end %>