helix
/
invidious
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Overhaul extractors.cr to use modules

Browse Source
pull/2228/head
syeopite 3 years ago
parent 3dea670091
commit 142317c2be
No known key found for this signature in database
GPG Key ID: 6FA616E5A5294A82
1 changed files with 255 additions and 273 deletions
Show Stats Download Patch File Download Diff File

528
src/invidious/helpers/extractors.cr
Unescape Escape View File

@ -3,257 +3,245 @@
# Tuple of Parsers/Extractors so we can easily cycle through them. # Tuple of Parsers/Extractors so we can easily cycle through them.
private ITEM_CONTAINER_EXTRACTOR = { private ITEM_CONTAINER_EXTRACTOR = {
YoutubeTabsExtractor.new, Extractors::YouTubeTabs,
SearchResultsExtractor.new, Extractors::SearchResults,
ContinuationExtractor.new, Extractors::Continuation,
} }
private ITEM_PARSERS = { private ITEM_PARSERS = {
VideoParser.new, Parsers::VideoRendererParser,
ChannelParser.new, Parsers::ChannelRendererParser,
GridPlaylistParser.new, Parsers::GridPlaylistRendererParser,
PlaylistParser.new, Parsers::PlaylistRendererParser,
CategoryParser.new, Parsers::CategoryRendererParser,
} }
private struct AuthorFallback record AuthorFallback, name : String? = nil, id : String? = nil
property name, id
def initialize(@name : String? = nil, @id : String? = nil)
end
end
# The following are the parsers for parsing raw item data into neatly packaged structs. # The following are the parsers for parsing raw item data into neatly packaged structs.
# They're accessed through the process() method which validates the given data as applicable # They're accessed through the process() method which validates the given data as applicable
# to their specific struct and then use the internal parse() method to assemble the struct # to their specific struct and then use the internal parse() method to assemble the struct
# specific to their category. # specific to their category.
private abstract struct ItemParser private module Parsers
# Base type for all item parsers. module VideoRendererParser
def process(item : JSON::Any, author_fallback : AuthorFallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
end if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?)
return self.parse(item_contents, author_fallback)
end
end
private def parse(item_contents : JSON::Any, author_fallback : AuthorFallback) private def self.parse(item_contents, author_fallback)
end video_id = item_contents["videoId"].as_s
end title = item_contents["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || ""
author_info = item_contents["ownerText"]?.try &.["runs"]?.try &.as_a?.try &.[0]?
author = author_info.try &.["text"].as_s || author_fallback.name || ""
author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_fallback.id || ""
published = item_contents["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local
view_count = item_contents["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
length_seconds = item_contents["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } ||
item_contents["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]?
.try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0
live_now = false
paid = false
premium = false
premiere_timestamp = item_contents["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) }
item_contents["badges"]?.try &.as_a.each do |badge|
b = badge["metadataBadgeRenderer"]
case b["label"].as_s
when "LIVE NOW"
live_now = true
when "New", "4K", "CC"
# TODO
when "Premium"
# TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"]
premium = true
else nil # Ignore
end
end
private struct VideoParser < ItemParser SearchVideo.new({
def process(item, author_fallback) title: title,
if item_contents = (item["videoRenderer"]? || item["gridVideoRenderer"]?) id: video_id,
return self.parse(item_contents, author_fallback) author: author,
ucid: author_id,
published: published,
views: view_count,
description_html: description_html,
length_seconds: length_seconds,
live_now: live_now,
premium: premium,
premiere_timestamp: premiere_timestamp,
})
end end
end end
private def parse(item_contents, author_fallback) module ChannelRendererParser
video_id = item_contents["videoId"].as_s def self.process(item : JSON::Any, author_fallback : AuthorFallback)
title = item_contents["title"].try { |t| t["simpleText"]?.try &.as_s || t["runs"]?.try &.as_a.map(&.["text"].as_s).join("") } || "" if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?)
return self.parse(item_contents, author_fallback)
author_info = item_contents["ownerText"]?.try &.["runs"]?.try &.as_a?.try &.[0]?
author = author_info.try &.["text"].as_s || author_fallback.name || ""
author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_fallback.id || ""
published = item_contents["publishedTimeText"]?.try &.["simpleText"]?.try { |t| decode_date(t.as_s) } || Time.local
view_count = item_contents["viewCountText"]?.try &.["simpleText"]?.try &.as_s.gsub(/\D+/, "").to_i64? || 0_i64
description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
length_seconds = item_contents["lengthText"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } ||
item_contents["thumbnailOverlays"]?.try &.as_a.find(&.["thumbnailOverlayTimeStatusRenderer"]?).try &.["thumbnailOverlayTimeStatusRenderer"]?
.try &.["text"]?.try &.["simpleText"]?.try &.as_s.try { |t| decode_length_seconds(t) } || 0
live_now = false
paid = false
premium = false
premiere_timestamp = item_contents["upcomingEventData"]?.try &.["startTime"]?.try { |t| Time.unix(t.as_s.to_i64) }
item_contents["badges"]?.try &.as_a.each do |badge|
b = badge["metadataBadgeRenderer"]
case b["label"].as_s
when "LIVE NOW"
live_now = true
when "New", "4K", "CC"
# TODO
when "Premium"
# TODO: Potentially available as item_contents["topStandaloneBadge"]["metadataBadgeRenderer"]
premium = true
else nil # Ignore
end end
end end
SearchVideo.new({ private def self.parse(item_contents, author_fallback)
title: title, author = item_contents["title"]["simpleText"]?.try &.as_s || author_fallback.name || ""
id: video_id, author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id || ""
author: author,
ucid: author_id, author_thumbnail = item_contents["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try &.["url"]?.try &.as_s || ""
published: published, subscriber_count = item_contents["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0
views: view_count,
description_html: description_html, auto_generated = false
length_seconds: length_seconds, auto_generated = true if !item_contents["videoCountText"]?
live_now: live_now, video_count = item_contents["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
premium: premium, description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
premiere_timestamp: premiere_timestamp,
}) SearchChannel.new({
end author: author,
end ucid: author_id,
author_thumbnail: author_thumbnail,
private struct ChannelParser < ItemParser subscriber_count: subscriber_count,
def process(item, author_fallback) video_count: video_count,
if item_contents = (item["channelRenderer"]? || item["gridChannelRenderer"]?) description_html: description_html,
return self.parse(item_contents, author_fallback) auto_generated: auto_generated,
})
end end
end end
private def parse(item_contents, author_fallback) module GridPlaylistRendererParser
author = item_contents["title"]["simpleText"]?.try &.as_s || author_fallback.name || "" def self.process(item : JSON::Any, author_fallback : AuthorFallback)
author_id = item_contents["channelId"]?.try &.as_s || author_fallback.id || "" if item_contents = item["gridPlaylistRenderer"]?
return self.parse(item_contents, author_fallback)
author_thumbnail = item_contents["thumbnail"]["thumbnails"]?.try &.as_a[0]?.try &.["url"]?.try &.as_s || "" end
subscriber_count = item_contents["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s.try { |s| short_text_to_number(s.split(" ")[0]) } || 0
auto_generated = false
auto_generated = true if !item_contents["videoCountText"]?
video_count = item_contents["videoCountText"]?.try &.["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
description_html = item_contents["descriptionSnippet"]?.try { |t| parse_content(t) } || ""
SearchChannel.new({
author: author,
ucid: author_id,
author_thumbnail: author_thumbnail,
subscriber_count: subscriber_count,
video_count: video_count,
description_html: description_html,
auto_generated: auto_generated,
})
end
end
private struct GridPlaylistParser < ItemParser
def process(item, author_fallback)
if item_contents = item["gridPlaylistRenderer"]?
return self.parse(item_contents, author_fallback)
end end
end
private def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
title = item_contents["title"]["runs"].as_a[0]?.try &.["text"].as_s || "" title = item_contents["title"]["runs"].as_a[0]?.try &.["text"].as_s || ""
plid = item_contents["playlistId"]?.try &.as_s || "" plid = item_contents["playlistId"]?.try &.as_s || ""
video_count = item_contents["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0 video_count = item_contents["videoCountText"]["runs"].as_a[0]?.try &.["text"].as_s.gsub(/\D/, "").to_i || 0
playlist_thumbnail = item_contents["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || "" playlist_thumbnail = item_contents["thumbnail"]["thumbnails"][0]?.try &.["url"]?.try &.as_s || ""
SearchPlaylist.new({ SearchPlaylist.new({
title: title, title: title,
id: plid, id: plid,
author: author_fallback.name || "", author: author_fallback.name || "",
ucid: author_fallback.id || "", ucid: author_fallback.id || "",
video_count: video_count, video_count: video_count,
videos: [] of SearchPlaylistVideo, videos: [] of SearchPlaylistVideo,
thumbnail: playlist_thumbnail, thumbnail: playlist_thumbnail,
}) })
end
end end
end
private struct PlaylistParser < ItemParser module PlaylistRendererParser
def process(item, author_fallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["playlistRenderer"]? if item_contents = item["playlistRenderer"]?
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end
end end
end
def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
title = item_contents["title"]["simpleText"]?.try &.as_s || "" title = item_contents["title"]["simpleText"]?.try &.as_s || ""
plid = item_contents["playlistId"]?.try &.as_s || "" plid = item_contents["playlistId"]?.try &.as_s || ""
video_count = item_contents["videoCount"]?.try &.as_s.to_i || 0 video_count = item_contents["videoCount"]?.try &.as_s.to_i || 0
playlist_thumbnail = item_contents["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || "" playlist_thumbnail = item_contents["thumbnails"].as_a[0]?.try &.["thumbnails"]?.try &.as_a[0]?.try &.["url"].as_s || ""
author_info = item_contents["shortBylineText"]?.try &.["runs"]?.try &.as_a?.try &.[0]? author_info = item_contents["shortBylineText"]?.try &.["runs"]?.try &.as_a?.try &.[0]?
author = author_info.try &.["text"].as_s || author_fallback.name || "" author = author_info.try &.["text"].as_s || author_fallback.name || ""
author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_fallback.id || "" author_id = author_info.try &.["navigationEndpoint"]?.try &.["browseEndpoint"]["browseId"].as_s || author_fallback.id || ""
videos = item_contents["videos"]?.try &.as_a.map do |v| videos = item_contents["videos"]?.try &.as_a.map do |v|
v = v["childVideoRenderer"] v = v["childVideoRenderer"]
v_title = v["title"]["simpleText"]?.try &.as_s || "" v_title = v["title"]["simpleText"]?.try &.as_s || ""
v_id = v["videoId"]?.try &.as_s || "" v_id = v["videoId"]?.try &.as_s || ""
v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0 v_length_seconds = v["lengthText"]?.try &.["simpleText"]?.try { |t| decode_length_seconds(t.as_s) } || 0
SearchPlaylistVideo.new({ SearchPlaylistVideo.new({
title: v_title, title: v_title,
id: v_id, id: v_id,
length_seconds: v_length_seconds, length_seconds: v_length_seconds,
})
end || [] of SearchPlaylistVideo
# TODO: item_contents["publishedTimeText"]?
SearchPlaylist.new({
title: title,
id: plid,
author: author,
ucid: author_id,
video_count: video_count,
videos: videos,
thumbnail: playlist_thumbnail,
}) })
end || [] of SearchPlaylistVideo end
# TODO: item_contents["publishedTimeText"]?
SearchPlaylist.new({
title: title,
id: plid,
author: author,
ucid: author_id,
video_count: video_count,
videos: videos,
thumbnail: playlist_thumbnail,
})
end end
end
private struct CategoryParser < ItemParser module CategoryRendererParser
def process(item, author_fallback) def self.process(item : JSON::Any, author_fallback : AuthorFallback)
if item_contents = item["shelfRenderer"]? if item_contents = item["shelfRenderer"]?
return self.parse(item_contents, author_fallback) return self.parse(item_contents, author_fallback)
end
end end
end
def parse(item_contents, author_fallback) private def self.parse(item_contents, author_fallback)
# Title extraction is a bit complicated. There are two possible routes for it # Title extraction is a bit complicated. There are two possible routes for it
# as well as times when the title attribute just isn't sent by YT. # as well as times when the title attribute just isn't sent by YT.
title_container = item_contents["title"]? || "" title_container = item_contents["title"]? || ""
if !title_container.is_a? String if !title_container.is_a? String
if title = title_container["simpleText"]? if title = title_container["simpleText"]?
title = title.as_s title = title.as_s
else
title = title_container["runs"][0]["text"].as_s
end
else else
title = title_container["runs"][0]["text"].as_s title = ""
end end
else
title = ""
end
url = item_contents["endpoint"]?.try &.["commandMetadata"]["webCommandMetadata"]["url"].as_s url = item_contents["endpoint"]?.try &.["commandMetadata"]["webCommandMetadata"]["url"].as_s
# Sometimes a category can have badges. # Sometimes a category can have badges.
badges = [] of Tuple(String, String) # (Badge style, label) badges = [] of Tuple(String, String) # (Badge style, label)
item_contents["badges"]?.try &.as_a.each do |badge| item_contents["badges"]?.try &.as_a.each do |badge|
badge = badge["metadataBadgeRenderer"] badge = badge["metadataBadgeRenderer"]
badges << {badge["style"].as_s, badge["label"].as_s} badges << {badge["style"].as_s, badge["label"].as_s}
end end
# Category description # Category description
description_html = item_contents["subtitle"]?.try { |desc| parse_content(desc) } || "" description_html = item_contents["subtitle"]?.try { |desc| parse_content(desc) } || ""
# Content parsing # Content parsing
contents = [] of SearchItem contents = [] of SearchItem
# Content could be in three locations. # Content could be in three locations.
if content_container = item_contents["content"]["horizontalListRenderer"]? if content_container = item_contents["content"]["horizontalListRenderer"]?
elsif content_container = item_contents["content"]["expandedShelfContentsRenderer"]? elsif content_container = item_contents["content"]["expandedShelfContentsRenderer"]?
elsif content_container = item_contents["content"]["verticalListRenderer"]? elsif content_container = item_contents["content"]["verticalListRenderer"]?
else else
content_container = item_contents["contents"] content_container = item_contents["contents"]
end end
raw_contents = content_container["items"].as_a raw_contents = content_container["items"].as_a
raw_contents.each do |item| raw_contents.each do |item|
result = extract_item(item) result = extract_item(item)
if !result.nil? if !result.nil?
contents << result contents << result
end
end end
end
Category.new({ Category.new({
title: title, title: title,
contents: contents, contents: contents,
description_html: description_html, description_html: description_html,
url: url, url: url,
badges: badges, badges: badges,
}) })
end
end end
end end
@ -262,88 +250,82 @@ end
# a structure we can more easily use via the parsers above. Their internals are # a structure we can more easily use via the parsers above. Their internals are
# identical to the item parsers. # identical to the item parsers.
private abstract struct ItemsContainerExtractor private module Extractors
def process(item : Hash(String, JSON::Any)) module YouTubeTabs
end def self.process(initial_data : Hash(String, JSON::Any))
if target = initial_data["twoColumnBrowseResultsRenderer"]?
private def extract(target : JSON::Any) self.extract(target)
end end
end
private struct YoutubeTabsExtractor < ItemsContainerExtractor
def process(initial_data)
if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target)
end end
end
private def extract(target)
raw_items = [] of JSON::Any
selected_tab = extract_selected_tab(target["tabs"])
content = selected_tab["content"]
content["sectionListRenderer"]["contents"].as_a.each do |renderer_container|
renderer_container = renderer_container["itemSectionRenderer"]
renderer_container_contents = renderer_container["contents"].as_a[0]
# Category extraction private def self.extract(target)
if items_container = renderer_container_contents["shelfRenderer"]? raw_items = [] of JSON::Any
raw_items << renderer_container_contents selected_tab = extract_selected_tab(target["tabs"])
next content = selected_tab["content"]
elsif items_container = renderer_container_contents["gridRenderer"]?
else content["sectionListRenderer"]["contents"].as_a.each do |renderer_container|
items_container = renderer_container_contents renderer_container = renderer_container["itemSectionRenderer"]
end renderer_container_contents = renderer_container["contents"].as_a[0]
# Category extraction
if items_container = renderer_container_contents["shelfRenderer"]?
raw_items << renderer_container_contents
next
elsif items_container = renderer_container_contents["gridRenderer"]?
else
items_container = renderer_container_contents
end
items_container["items"].as_a.each do |item| items_container["items"].as_a.each do |item|
raw_items << item raw_items << item
end
end end
end
return raw_items
end
end
private struct SearchResultsExtractor < ItemsContainerExtractor return raw_items
def process(initial_data)
if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target)
end end
end end
private def extract(target) module SearchResults
raw_items = [] of Array(JSON::Any) def self.process(initial_data : Hash(String, JSON::Any))
content = target["primaryContents"] if target = initial_data["twoColumnSearchResultsRenderer"]?
renderer = content["sectionListRenderer"]["contents"].as_a.each do |node| self.extract(target)
if node = node["itemSectionRenderer"]?
raw_items << node["contents"].as_a
end end
end end
raw_items = raw_items.flatten private def self.extract(target)
raw_items = [] of Array(JSON::Any)
content = target["primaryContents"]
renderer = content["sectionListRenderer"]["contents"].as_a.each do |node|
if node = node["itemSectionRenderer"]?
raw_items << node["contents"].as_a
end
end
return raw_items raw_items = raw_items.flatten
end
end
private struct ContinuationExtractor < ItemsContainerExtractor return raw_items
def process(initial_data)
if target = initial_data["continuationContents"]?
self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
self.extract(target)
end end
end end
private def extract(target) module Continuation
raw_items = [] of JSON::Any def self.process(initial_data : Hash(String, JSON::Any))
if content = target["gridContinuation"]? if target = initial_data["continuationContents"]?
raw_items = content["items"].as_a self.extract(target)
elsif content = target["continuationItems"]? elsif target = initial_data["appendContinuationItemsAction"]?
raw_items = content.as_a self.extract(target)
end
end end
return raw_items private def self.extract(target)
raw_items = [] of JSON::Any
if content = target["gridContinuation"]?
raw_items = content["items"].as_a
elsif content = target["continuationItems"]?
raw_items = content.as_a
end
return raw_items
end
end end
end end

Write Preview
Loading…
Cancel
Save