extractors: separate 'extract' and 'parse' logic

pull/3419/head
Samantaz Fox 2 years ago
parent 9588fcb5d1
commit bdc51cd20f
No known key found for this signature in database
GPG Key ID: F42821059186176E

@ -8,7 +8,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
extract_item(item, author, ucid).try { |t| items << t }
parse_item(item, author, ucid).try { |t| items << t }
}
continuation = continuation_items.as_a.last["continuationItemRenderer"]?

@ -37,7 +37,7 @@ module Invidious::Search
items = [] of SearchItem
continuation_items.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each do |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
parse_item(item["itemSectionRenderer"]["contents"].as_a[0]).try { |t| items << t }
end
return items

@ -20,6 +20,8 @@ private ITEM_PARSERS = {
Parsers::ReelItemRendererParser,
}
private alias InitialData = Hash(String, JSON::Any)
record AuthorFallback, name : String, id : String
# Namespace for logic relating to parsing InnerTube data into various datastructs.
@ -348,7 +350,7 @@ private module Parsers
raw_contents = content_container["items"]?.try &.as_a
if !raw_contents.nil?
raw_contents.each do |item|
result = extract_item(item)
result = parse_item(item)
if !result.nil?
contents << result
end
@ -510,7 +512,7 @@ private module Extractors
# }]
#
module YouTubeTabs
def self.process(initial_data : Hash(String, JSON::Any))
def self.process(initial_data : InitialData)
if target = initial_data["twoColumnBrowseResultsRenderer"]?
self.extract(target)
end
@ -575,7 +577,7 @@ private module Extractors
# }
#
module SearchResults
def self.process(initial_data : Hash(String, JSON::Any))
def self.process(initial_data : InitialData)
if target = initial_data["twoColumnSearchResultsRenderer"]?
self.extract(target)
end
@ -608,8 +610,8 @@ private module Extractors
# The way they are structured is too varied to be accurately written down here.
# However, they all eventually lead to an array of parsable items after traversing
# through the JSON structure.
module Continuation
def self.process(initial_data : Hash(String, JSON::Any))
module ContinuationContent
def self.process(initial_data : InitialData)
if target = initial_data["continuationContents"]?
self.extract(target)
elsif target = initial_data["appendContinuationItemsAction"]?
@ -691,8 +693,7 @@ end
# Parses an item from Youtube's JSON response into a more usable structure.
# The end result can either be a SearchVideo, SearchPlaylist or SearchChannel.
def extract_item(item : JSON::Any, author_fallback : String? = "",
author_id_fallback : String? = "")
def parse_item(item : JSON::Any, author_fallback : String? = "", author_id_fallback : String? = "")
# We "allow" nil values but secretly use empty strings instead. This is to save us the
# hassle of modifying every author_fallback and author_id_fallback arg usage
# which is more often than not nil.
@ -702,24 +703,23 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
# Each parser automatically validates the data given to see if the data is
# applicable to itself. If not nil is returned and the next parser is attempted.
ITEM_PARSERS.each do |parser|
LOGGER.trace("extract_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
LOGGER.trace("parse_item: Attempting to parse item using \"#{parser.parser_name}\" (cycling...)")
if result = parser.process(item, author_fallback)
LOGGER.debug("extract_item: Successfully parsed via #{parser.parser_name}")
LOGGER.debug("parse_item: Successfully parsed via #{parser.parser_name}")
return result
else
LOGGER.trace("extract_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
LOGGER.trace("parse_item: Parser \"#{parser.parser_name}\" does not apply. Cycling to the next one...")
end
end
end
# Parses multiple items from YouTube's initial JSON response into a more usable structure.
# The end result is an array of SearchItem.
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil) : Array(SearchItem)
items = [] of SearchItem
#
# This function yields the container so that items can be parsed separately.
#
def extract_items(initial_data : InitialData, &block)
if unpackaged_data = initial_data["contents"]?.try &.as_h
elsif unpackaged_data = initial_data["response"]?.try &.as_h
elsif unpackaged_data = initial_data.dig?("onResponseReceivedActions", 0).try &.as_h
@ -727,24 +727,32 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
unpackaged_data = initial_data
end
# This is identical to the parser cycling of extract_item().
# This is identical to the parser cycling of parse_item().
ITEM_CONTAINER_EXTRACTOR.each do |extractor|
LOGGER.trace("extract_items: Attempting to extract item container using \"#{extractor.extractor_name}\" (cycling...)")
if container = extractor.process(unpackaged_data)
LOGGER.debug("extract_items: Successfully unpacked container with \"#{extractor.extractor_name}\"")
# Extract items in container
container.each do |item|
if parsed_result = extract_item(item, author_fallback, author_id_fallback)
items << parsed_result
end
end
break
container.each { |item| yield item }
else
LOGGER.trace("extract_items: Extractor \"#{extractor.extractor_name}\" does not apply. Cycling to the next one...")
end
end
end
# Wrapper using the block function above
def extract_items(
initial_data : InitialData,
author_fallback : String? = nil,
author_id_fallback : String? = nil
) : Array(SearchItem)
items = [] of SearchItem
extract_items(initial_data) do |item|
parsed = parse_item(item, author_fallback, author_id_fallback)
items << parsed if !parsed.nil?
end
return items
end

Loading…
Cancel
Save