Don't overwrite published date for channel_videos

src/invidious/channels.cr

@@ -102,76 +102,72 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
     auto_generated = true
   end
 
-  if !pull_all_videos
+  page = 1
-    url = produce_channel_videos_url(ucid, 1, auto_generated: auto_generated)
-    response = client.get(url)
-    json = JSON.parse(response.body)
 
-    if json["content_html"]? && !json["content_html"].as_s.empty?
+  url = produce_channel_videos_url(ucid, page, auto_generated: auto_generated)
-      document = XML.parse_html(json["content_html"].as_s)
+  response = client.get(url)
-      nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
+  json = JSON.parse(response.body)
 
-      if auto_generated
+  if json["content_html"]? && !json["content_html"].as_s.empty?
-        videos = extract_videos(nodeset)
+    document = XML.parse_html(json["content_html"].as_s)
-      else
+    nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-        videos = extract_videos(nodeset, ucid, author)
+
-      end
+    if auto_generated
+      videos = extract_videos(nodeset)
+    else
+      videos = extract_videos(nodeset, ucid, author)
     end
+  end
 
-    videos ||= [] of ChannelVideo
+  videos ||= [] of ChannelVideo
 
-    rss.xpath_nodes("//feed/entry").each do |entry|
+  rss.xpath_nodes("//feed/entry").each do |entry|
-      video_id = entry.xpath_node("videoid").not_nil!.content
+    video_id = entry.xpath_node("videoid").not_nil!.content
-      title = entry.xpath_node("title").not_nil!.content
+    title = entry.xpath_node("title").not_nil!.content
-      published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
+    published = Time.parse_rfc3339(entry.xpath_node("published").not_nil!.content)
-      updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
+    updated = Time.parse_rfc3339(entry.xpath_node("updated").not_nil!.content)
-      author = entry.xpath_node("author/name").not_nil!.content
+    author = entry.xpath_node("author/name").not_nil!.content
-      ucid = entry.xpath_node("channelid").not_nil!.content
+    ucid = entry.xpath_node("channelid").not_nil!.content
 
-      channel_video = videos.select { |video| video.id == video_id }[0]?
+    channel_video = videos.select { |video| video.id == video_id }[0]?
 
-      length_seconds = channel_video.try &.length_seconds
+    length_seconds = channel_video.try &.length_seconds
-      length_seconds ||= 0
+    length_seconds ||= 0
 
-      live_now = channel_video.try &.live_now
+    live_now = channel_video.try &.live_now
-      live_now ||= false
+    live_now ||= false
 
-      premiere_timestamp = channel_video.try &.premiere_timestamp
+    premiere_timestamp = channel_video.try &.premiere_timestamp
 
-      # Deliver notifications to `/api/v1/auth/notifications`
+    video = ChannelVideo.new(
-      # payload = {
+      id: video_id,
-      #   "key"   => video_id,
+      title: title,
-      #   "topic" => ucid,
+      published: published,
-      # }.to_json
+      updated: Time.now,
-      # PG_DB.exec("NOTIFY notifications, E'#{payload}'")
+      ucid: ucid,
+      author: author,
+      length_seconds: length_seconds,
+      live_now: live_now,
+      premiere_timestamp: premiere_timestamp
+    )
 
-      video = ChannelVideo.new(
+    db.exec("UPDATE users SET notifications = notifications || $1 \
-        id: video_id,
-        title: title,
-        published: published,
-        updated: Time.now,
-        ucid: ucid,
-        author: author,
-        length_seconds: length_seconds,
-        live_now: live_now,
-        premiere_timestamp: premiere_timestamp
-      )
-
-      db.exec("UPDATE users SET notifications = notifications || $1 \
         WHERE updated < $2 AND $3 = ANY(subscriptions) AND $1 <> ALL(notifications)", video.id, video.published, ucid)
 
-      video_array = video.to_a
+    video_array = video.to_a
-      args = arg_array(video_array)
+    args = arg_array(video_array)
 
-      # We don't include the 'premire_timestamp' here because channel pages don't include them,
+    # We don't include the 'premire_timestamp' here because channel pages don't include them,
-      # meaning the above timestamp is always null
+    # meaning the above timestamp is always null
-      db.exec("INSERT INTO channel_videos VALUES (#{args}) \
+    db.exec("INSERT INTO channel_videos VALUES (#{args}) \
       ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
       updated = $4, ucid = $5, author = $6, length_seconds = $7, \
       live_now = $8", video_array)
-    end
+  end
-  else
+
-    page = 1
+  if pull_all_videos
+    page += 1
+
     ids = [] of String
 
     loop do
@@ -186,6 +182,8 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
         break
       end
 
+      nodeset = nodeset.not_nil!
+
       if auto_generated
         videos = extract_videos(nodeset)
       else
@@ -216,16 +214,14 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
           video_array = video.to_a
           args = arg_array(video_array)
 
-          # We don't include the 'premire_timestamp' here because channel pages don't include them,
+          # We don't update the 'premire_timestamp' here because channel pages don't include them
-          # meaning the above timestamp is always null
           db.exec("INSERT INTO channel_videos VALUES (#{args}) \
-          ON CONFLICT (id) DO UPDATE SET title = $2, published = $3, \
+          ON CONFLICT (id) DO UPDATE SET title = $2, updated = $4, \
-          updated = $4, ucid = $5, author = $6, length_seconds = $7, \
+          ucid = $5, author = $6, length_seconds = $7, live_now = $8", video_array)
-          live_now = $8", video_array)
         end
       end
 
-      if count < 30
+      if count < 25
         break
       end