Videos: parse data during first fetching

There will be less data to be stores in the DB cache

spec/invidious/videos/scheduled_live_extract_spec.cr

@@ -22,7 +22,6 @@ Spectator.describe Invidious::Hashtag do
     expect(info["likes"].as_i).to eq(2_283)
 
     expect(info["genre"].as_s).to eq("Gaming")
-    expect(info["genreUrl"].raw).to be_nil
     expect(info["genreUcid"].as_s).to be_empty
     expect(info["license"].as_s).to be_empty
 
@@ -81,7 +80,6 @@ Spectator.describe Invidious::Hashtag do
     expect(info["likes"].as_i).to eq(22)
 
     expect(info["genre"].as_s).to eq("Entertainment")
-    expect(info["genreUrl"].raw).to be_nil
     expect(info["genreUcid"].as_s).to be_empty
     expect(info["license"].as_s).to be_empty
 

src/invidious/videos.cr

@@ -1,3 +1,9 @@
+enum VideoType
+  Video
+  Livestream
+  Scheduled
+end
+
 struct Video
   include DB::Serializable
 
@@ -27,7 +33,7 @@ struct Video
 
   def to_json(locale : String?, json : JSON::Builder)
     json.object do
-      json.field "type", "video"
+      json.field "type", self.video_type
 
       json.field "title", self.title
       json.field "videoId", self.id
@@ -253,61 +259,22 @@ struct Video
     to_json(nil, json)
   end
 
-  def title
+  def video_type : VideoType
-    info["videoDetails"]["title"]?.try &.as_s || ""
+    video_type = info["videoType"]?.try &.as_s || "video"
-  end
+    return VideoType.parse?(video_type) || VideoType::Video
-
-  def ucid
-    info["videoDetails"]["channelId"]?.try &.as_s || ""
-  end
-
-  def author
-    info["videoDetails"]["author"]?.try &.as_s || ""
-  end
-
-  def length_seconds : Int32
-    info.dig?("microformat", "playerMicroformatRenderer", "lengthSeconds").try &.as_s.to_i ||
-      info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
-  end
-
-  def views : Int64
-    info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
-  end
-
-  def likes : Int64
-    info["likes"]?.try &.as_i64 || 0_i64
-  end
-
-  def dislikes : Int64
-    info["dislikes"]?.try &.as_i64 || 0_i64
   end
 
   def published : Time
-    info
+    return info["published"]?
-      .dig?("microformat", "playerMicroformatRenderer", "publishDate")
       .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
   end
 
   def published=(other : Time)
-    info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
+    info["published"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
-  end
-
-  def allow_ratings
-    r = info["videoDetails"]["allowRatings"]?.try &.as_bool
-    r.nil? ? false : r
   end
 
   def live_now
-    info["microformat"]?.try &.["playerMicroformatRenderer"]?
+    return (self.video_type == VideoType::Livestream)
-      .try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
-  end
-
-  def is_listed
-    info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
-  end
-
-  def is_upcoming
-    info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
   end
 
   def premiere_timestamp : Time?
@@ -316,31 +283,11 @@ struct Video
       .try { |t| Time.parse_rfc3339(t.as_s) }
   end
 
-  def keywords
-    info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
-  end
-
   def related_videos
     info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
   end
 
-  def allowed_regions
+  # Methods for parsing streaming data
-    info
-      .dig?("microformat", "playerMicroformatRenderer", "availableCountries")
-      .try &.as_a.map &.as_s || [] of String
-  end
-
-  def author_thumbnail : String
-    info["authorThumbnail"]?.try &.as_s || ""
-  end
-
-  def author_verified : Bool
-    info["authorVerified"]?.try &.as_bool || false
-  end
-
-  def sub_count_text : String
-    info["subCountText"]?.try &.as_s || "-"
-  end
 
   def fmt_stream
     return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
@@ -391,6 +338,8 @@ struct Video
     adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
   end
 
+  # Misc. methods
+
   def storyboards
     storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
       .try &.as_s.split("|")
@@ -454,8 +403,7 @@ struct Video
   end
 
   def paid
-    reason = info.dig?("playabilityStatus", "reason").try &.as_s || ""
+    return (self.reason || "").includes? "requires payment"
-    return reason.includes? "requires payment"
   end
 
   def premium
@@ -470,65 +418,114 @@ struct Video
     return @captions
   end
 
-  def description
+  def hls_manifest_url : String?
-    description = info
+    info.dig?("streamingData", "hlsManifestUrl").try &.as_s
-      .dig?("microformat", "playerMicroformatRenderer", "description", "simpleText")
-      .try &.as_s || ""
   end
 
-  # TODO
+  def dash_manifest_url
-  def description=(value : String)
+    info.dig?("streamingData", "dashManifestUrl").try &.as_s
-    @description = value
   end
 
-  def description_html
+  def genre_url : String?
-    info["descriptionHtml"]?.try &.as_s || "<p></p>"
+    info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
   end
 
-  def description_html=(value : String)
+  def is_vr : Bool?
-    info["descriptionHtml"] = JSON::Any.new(value)
+    return {"EQUIRECTANGULAR", "MESH"}.includes? self.projection_type
   end
 
-  def short_description
+  def projection_type : String?
-    info["shortDescription"]?.try &.as_s? || ""
+    return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
   end
 
-  def hls_manifest_url : String?
+  def reason : String?
-    info.dig?("streamingData", "hlsManifestUrl").try &.as_s
+    info["reason"]?.try &.as_s
   end
 
-  def dash_manifest_url
+  # Macros defining getters/setters for various types of data
-    info.dig?("streamingData", "dashManifestUrl").try &.as_s
+
+  private macro getset_string(name)
+    # Return {{name.stringify}} from `info`
+    def {{name.id.underscore}} : String
+      return info[{{name.stringify}}]?.try &.as_s || ""
     end
 
-  def genre : String
+    # Update {{name.stringify}} into `info`
-    info["genre"]?.try &.as_s || ""
+    def {{name.id.underscore}}=(value : String)
+      info[{{name.stringify}}] = JSON::Any.new(value)
     end
 
-  def genre_url : String?
+    {% if flag?(:debug_macros) %} {{debug}} {% end %}
-    info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
   end
 
-  def license : String?
+  private macro getset_string_array(name)
-    info["license"]?.try &.as_s
+    # Return {{name.stringify}} from `info`
+    def {{name.id.underscore}} : Array(String)
+      return info[{{name.stringify}}]?.try &.as_a.map &.as_s || [] of String
     end
 
-  def is_family_friendly : Bool
+    # Update {{name.stringify}} into `info`
-    info.dig?("microformat", "playerMicroformatRenderer", "isFamilySafe").try &.as_bool || false
+    def {{name.id.underscore}}=(value : Array(String))
+      info[{{name.stringify}}] = JSON::Any.new(value)
     end
 
-  def is_vr : Bool?
+    {% if flag?(:debug_macros) %} {{debug}} {% end %}
-    projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
-    return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
   end
 
-  def projection_type : String?
+  {% for op, type in {i32: Int32, i64: Int64} %}
-    return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
+    private macro getset_{{op}}(name)
+      def \{{name.id.underscore}} : {{type}}
+        return info[\{{name.stringify}}]?.try &.as_i.to_{{op}} || 0_{{op}}
       end
 
-  def reason : String?
+      def \{{name.id.underscore}}=(value : Int)
-    info["reason"]?.try &.as_s
+        info[\{{name.stringify}}] = JSON::Any.new(value.to_i64)
       end
+
+      \{% if flag?(:debug_macros) %} \{{debug}} \{% end %}
+    end
+  {% end %}
+
+  private macro getset_bool(name)
+    # Return {{name.stringify}} from `info`
+    def {{name.id.underscore}} : Bool
+      return info[{{name.stringify}}]?.try &.as_bool || false
+    end
+
+    # Update {{name.stringify}} into `info`
+    def {{name.id.underscore}}=(value : Bool)
+      info[{{name.stringify}}] = JSON::Any.new(value)
+    end
+
+    {% if flag?(:debug_macros) %} {{debug}} {% end %}
+  end
+
+  # Method definitions, using the macros above
+
+  getset_string author
+  getset_string authorThumbnail
+  getset_string description
+  getset_string descriptionHtml
+  getset_string genre
+  getset_string genreUcid
+  getset_string license
+  getset_string shortDescription
+  getset_string subCountText
+  getset_string title
+  getset_string ucid
+
+  getset_string_array allowedRegions
+  getset_string_array keywords
+
+  getset_i32 lengthSeconds
+  getset_i64 likes
+  getset_i64 views
+
+  getset_bool allowRatings
+  getset_bool authorVerified
+  getset_bool isFamilyFriendly
+  getset_bool isListed
+  getset_bool isUpcoming
 end
 
 class VideoRedirect < Exception
@@ -684,6 +681,42 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
   raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
   raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
 
+  video_details = player_response.dig?("videoDetails")
+  microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
+
+  raise BrokenTubeException.new("videoDetails") if !video_details
+  raise BrokenTubeException.new("microformat") if !microformat
+
+  # Basic video infos
+
+  title = video_details["title"]?.try &.as_s
+  views = video_details["viewCount"]?.try &.as_s.to_i64
+
+  length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
+    .try &.as_s.to_i64
+
+  published = microformat["publishDate"]?
+    .try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
+
+  premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
+    .try { |t| Time.parse_rfc3339(t.as_s) }
+
+  live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
+    .try &.as_bool || false
+
+  # Extra video infos
+
+  allowed_regions = microformat["availableCountries"]?
+    .try &.as_a.map &.as_s || [] of String
+
+  allow_ratings = video_details["allowRatings"]?.try &.as_bool
+  family_friendly = microformat["isFamilySafe"].try &.as_bool
+  is_listed = video_details["isCrawlable"]?.try &.as_bool
+  is_upcoming = video_details["isUpcoming"]?.try &.as_bool
+
+  keywords = video_details["keywords"]?
+    .try &.as_a.map &.as_s || [] of String
+
   # Related videos
 
   LOGGER.debug("extract_video_info: parsing related videos...")
@@ -738,6 +771,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
 
   # Description
 
+  description = microformat.dig?("description", "simpleText").try &.as_s || ""
   short_description = player_response.dig?("videoDetails", "shortDescription")
 
   description_html = video_secondary_renderer.try &.dig?("description", "runs")
@@ -749,7 +783,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
     .try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
       .try &.as_a
 
-  genre = player_response.dig?("microformat", "playerMicroformatRenderer", "category")
+  genre = microformat["category"]?
   genre_ucid = nil
   license = nil
 
@@ -771,6 +805,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
 
   # Author infos
 
+  author = video_details["author"]?.try &.as_s
+  ucid = video_details["channelId"]?.try &.as_s
+
   if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
     author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
     author_verified = has_verified_badge?(author_info["badges"]?)
@@ -782,16 +819,43 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any
 
   # Return data
 
+  if live_now
+    video_type = VideoType::Livestream
+  elsif premiere_timestamp.not_nil!
+    video_type = VideoType::Scheduled
+    published = premiere_timestamp || Time.utc
+  else
+    video_type = VideoType::Video
+  end
+
   params = {
-    "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+    "videoType" => JSON::Any.new(video_type.to_s),
-    "relatedVideos"    => JSON::Any.new(related),
+    # Basic video infos
+    "title"         => JSON::Any.new(title || ""),
+    "views"         => JSON::Any.new(views || 0_i64),
     "likes"         => JSON::Any.new(likes || 0_i64),
-    "dislikes"         => JSON::Any.new(0_i64),
+    "lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
+    "published"     => JSON::Any.new(published.to_rfc3339),
+    # Extra video infos
+    "allowedRegions"   => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
+    "allowRatings"     => JSON::Any.new(allow_ratings || false),
+    "isFamilyFriendly" => JSON::Any.new(family_friendly || false),
+    "isListed"         => JSON::Any.new(is_listed || false),
+    "isUpcoming"       => JSON::Any.new(is_upcoming || false),
+    "keywords"         => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
+    # Related videos
+    "relatedVideos" => JSON::Any.new(related),
+    # Description
+    "description"      => JSON::Any.new(description || ""),
     "descriptionHtml"  => JSON::Any.new(description_html || "<p></p>"),
+    "shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
+    # Video metadata
     "genre"     => JSON::Any.new(genre.try &.as_s || ""),
-    "genreUrl"         => JSON::Any.new(nil),
     "genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
     "license"   => JSON::Any.new(license.try &.as_s || ""),
+    # Author infos
+    "author"          => JSON::Any.new(author || ""),
+    "ucid"            => JSON::Any.new(ucid || ""),
     "authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
     "authorVerified"  => JSON::Any.new(author_verified),
     "subCountText"    => JSON::Any.new(subs_text || "-"),