helix/invidious
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Migrate more types to data_structs

Browse source
This commit is contained in:
syeopite 2021-08-08 16:34:30 -07:00
parent 2333221e14
commit df1e4888cd
No known key found for this signature in database
GPG key ID: 6FA616E5A5294A82
30 changed files with 142 additions and 1286 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

18
src/invidious.cr
View file

@ -113,17 +113,17 @@ LOGGER = Invidious::LogHandler.new(OUTPUT, CONFIG.log_level)
if CONFIG.check_tables if CONFIG.check_tables
check_enum(PG_DB, "privacy", PlaylistPrivacy) check_enum(PG_DB, "privacy", PlaylistPrivacy)
check_table(PG_DB, "channels", InvidiousChannel) check_table(PG_DB, "channels", InvidiousStructs::Channel)
check_table(PG_DB, "channel_videos", ChannelVideo) check_table(PG_DB, "channel_videos", InvidiousStructs::ChannelVideo)
check_table(PG_DB, "playlists", InvidiousPlaylist) check_table(PG_DB, "playlists", InvidiousStructs::Playlist)
check_table(PG_DB, "playlist_videos", PlaylistVideo) check_table(PG_DB, "playlist_videos", YouTubeStructs::PlaylistVideo)
check_table(PG_DB, "nonces", Nonce) check_table(PG_DB, "nonces", Nonce)
check_table(PG_DB, "session_ids", SessionId) check_table(PG_DB, "session_ids", SessionId)
check_table(PG_DB, "users", User) check_table(PG_DB, "users", User)
check_table(PG_DB, "videos", Video) check_table(PG_DB, "videos", YouTubeStructs::Video)
if CONFIG.cache_annotations if CONFIG.cache_annotations
check_table(PG_DB, "annotations", Annotation) check_table(PG_DB, "annotations", YouTubeStructs::Annotation)
end end
end end
@ -646,14 +646,14 @@ get "/subscription_manager" do |env|
values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}"
end end
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousStructs::Channel)
subscriptions.sort_by! { |channel| channel.author.downcase } subscriptions.sort_by! { |channel| channel.author.downcase }
if action_takeout if action_takeout
if format == "json" if format == "json"
env.response.content_type = "application/json" env.response.content_type = "application/json"
env.response.headers["content-disposition"] = "attachment" env.response.headers["content-disposition"] = "attachment"
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist)
next JSON.build do |json| next JSON.build do |json|
json.object do json.object do
@ -795,7 +795,7 @@ post "/data_control" do |env|
next next
end end
playlist_video = PlaylistVideo.new({ playlist_video = YouTubeStructs::PlaylistVideo.new({
title: video.title, title: video.title,
id: video.id, id: video.id,
author: video.author, author: video.author,

37
src/invidious/channels/about.cr
View file

@ -1,32 +1,3 @@
# TODO: Refactor into either SearchChannel or InvidiousChannel
struct AboutChannel
include DB::Serializable
property ucid : String
property author : String
property auto_generated : Bool
property author_url : String
property author_thumbnail : String
property banner : String?
property description_html : String
property total_views : Int64
property sub_count : Int32
property joined : Time
property is_family_friendly : Bool
property allowed_regions : Array(String)
property related_channels : Array(AboutRelatedChannel)
property tabs : Array(String)
end
struct AboutRelatedChannel
include DB::Serializable
property ucid : String
property author : String
property author_url : String
property author_thumbnail : String
end
def get_about_info(ucid, locale) def get_about_info(ucid, locale)
begin begin
# "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"}
@ -64,7 +35,7 @@ def get_about_info(ucid, locale)
is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool
allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s } allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map { |a| a.as_s }
related_channels = [] of AboutRelatedChannel related_channels = [] of YouTubeStructs::AboutRelatedChannel
else else
author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s
author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s
@ -109,14 +80,14 @@ def get_about_info(ucid, locale)
related_author_thumbnail ||= "" related_author_thumbnail ||= ""
end end
AboutRelatedChannel.new({ YouTubeStructs::AboutRelatedChannel.new({
ucid: related_id, ucid: related_id,
author: related_title, author: related_title,
author_url: related_author_url, author_url: related_author_url,
author_thumbnail: related_author_thumbnail, author_thumbnail: related_author_thumbnail,
}) })
end end
related_channels ||= [] of AboutRelatedChannel related_channels ||= [] of YouTubeStructs::AboutRelatedChannel
end end
total_views = 0_i64 total_views = 0_i64
@ -155,7 +126,7 @@ def get_about_info(ucid, locale)
sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s?
.try { |text| short_text_to_number(text.split(" ")[0]) } || 0 .try { |text| short_text_to_number(text.split(" ")[0]) } || 0
AboutChannel.new({ YouTubeStructs::AboutChannel.new({
ucid: ucid, ucid: ucid,
author: author, author: author,
auto_generated: auto_generated, auto_generated: auto_generated,

117
src/invidious/channels/channels.cr
View file

@ -1,112 +1,3 @@
struct InvidiousChannel
include DB::Serializable
property id : String
property author : String
property updated : Time
property deleted : Bool
property subscribed : Time?
end
struct ChannelVideo
include DB::Serializable
property id : String
property title : String
property published : Time
property updated : Time
property ucid : String
property author : String
property length_seconds : Int32 = 0
property live_now : Bool = false
property premiere_timestamp : Time? = nil
property views : Int64? = nil
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "shortVideo"
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "viewCount", self.views
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def to_xml(locale, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("updated") { xml.text self.updated.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
end
end
end
def to_xml(locale, xml : XML::Builder | Nil = nil)
if xml
to_xml(locale, xml)
else
XML.build do |xml|
to_xml(locale, xml)
end
end
end
def to_tuple
{% begin %}
{
{{*@type.instance_vars.map { |var| var.name }}}
}
{% end %}
end
end
class ChannelRedirect < Exception class ChannelRedirect < Exception
property channel_id : String property channel_id : String
@ -152,7 +43,7 @@ def get_batch_channels(channels, db, refresh = false, pull_all_videos = true, ma
end end
def get_channel(id, db, refresh = true, pull_all_videos = true) def get_channel(id, db, refresh = true, pull_all_videos = true)
if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousChannel) if channel = db.query_one?("SELECT * FROM channels WHERE id = $1", id, as: InvidiousStructs::Channel)
if refresh && Time.utc - channel.updated > 10.minutes if refresh && Time.utc - channel.updated > 10.minutes
channel = fetch_channel(id, db, pull_all_videos: pull_all_videos) channel = fetch_channel(id, db, pull_all_videos: pull_all_videos)
channel_array = channel.to_a channel_array = channel.to_a
@ -224,7 +115,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
premiere_timestamp = channel_video.try &.premiere_timestamp premiere_timestamp = channel_video.try &.premiere_timestamp
video = ChannelVideo.new({ video = InvidiousStructs::ChannelVideo.new({
id: video_id, id: video_id,
title: title, title: title,
published: published, published: published,
@ -265,7 +156,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
videos = extract_videos(initial_data, author, ucid) videos = extract_videos(initial_data, author, ucid)
count = videos.size count = videos.size
videos = videos.map { |video| ChannelVideo.new({ videos = videos.map { |video| InvidiousStructs::ChannelVideo.new({
id: video.id, id: video.id,
title: video.title, title: video.title,
published: video.published, published: video.published,
@ -299,7 +190,7 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil)
end end
end end
channel = InvidiousChannel.new({ channel = InvidiousStructs::Channel.new({
id: ucid, id: ucid,
author: author, author: author,
updated: Time.utc, updated: Time.utc,

6
src/invidious/channels/playlists.cr
View file

@ -4,9 +4,9 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
continuationItems = response_json["onResponseReceivedActions"]? continuationItems = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"] .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return [] of SearchItem, nil if !continuationItems return [] of YouTubeStructs::Renderer, nil if !continuationItems
items = [] of SearchItem items = [] of YouTubeStructs::Renderer
continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item| continuationItems.as_a.select(&.as_h.has_key?("gridPlaylistRenderer")).each { |item|
extract_item(item, author, ucid).try { |t| items << t } extract_item(item, author, ucid).try { |t| items << t }
} }
@ -28,7 +28,7 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by)
response = YT_POOL.client &.get(url) response = YT_POOL.client &.get(url)
initial_data = extract_initial_data(response.body) initial_data = extract_initial_data(response.body)
return [] of SearchItem, nil if !initial_data return [] of YouTubeStructs::Renderer, nil if !initial_data
items = extract_items(initial_data, author, ucid) items = extract_items(initial_data, author, ucid)
continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]? continuation = response.body.match(/"token":"(?<continuation>[^"]+)"/).try &.["continuation"]?

2
src/invidious/channels/videos.cr
View file

@ -65,7 +65,7 @@ def get_channel_videos_response(ucid, page = 1, auto_generated = nil, sort_by =
end end
def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest") def get_60_videos(ucid, author, page, auto_generated, sort_by = "newest")
videos = [] of SearchVideo videos = [] of YouTubeStructs::VideoRenderer
2.times do |i| 2.times do |i|
initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by) initial_data = get_channel_videos_response(ucid, page * 2 + (i - 1), auto_generated: auto_generated, sort_by: sort_by)

3
src/invidious/data_structs/invidious/channel.cr
View file

@ -1,11 +1,10 @@
# Data structs used by Invidious to provide certain features. # Data structs used by Invidious to provide certain features.
module InvidiousStructs module InvidiousStructs
# Struct for representing a cached YouTube channel. # Struct for representing a cached YouTube channel.
# #
# This is constructed from YouTube's RSS feeds for channels and is # This is constructed from YouTube's RSS feeds for channels and is
# currently only used for storing subscriptions in a user. # currently only used for storing subscriptions in a user.
struct InvidiousChannel struct Channel
include DB::Serializable include DB::Serializable
property id : String property id : String

7
src/invidious/data_structs/invidious/playlists.cr
View file

@ -1,5 +1,6 @@
module InvidiousStructs module InvidiousStructs
private module PlaylistPrivacyConverter # Converter to parse a Invidious privacy type string to enum
module PlaylistPrivacyConverter
def self.from_rs(rs) def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8)))) return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end end
@ -16,7 +17,7 @@ module InvidiousStructs
property created : Time property created : Time
property updated : Time property updated : Time
@[DB::Field(converter: PlaylistPrivacyConverter)] @[DB::Field(converter: InvidiousStructs::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64) property index : Array(Int64)
@ -25,7 +26,7 @@ module InvidiousStructs
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil) def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do json.object do
json.field "type", "invidiousPlaylist" json.field "type", "InvidiousStructs::Playlist"
json.field "title", self.title json.field "title", self.title
json.field "playlistId", self.id json.field "playlistId", self.id

0
src/invidious/data_structs/base.cr → src/invidious/data_structs/youtube/base.cr
View file

1
src/invidious/data_structs/youtube/channel.cr
View file

@ -14,7 +14,6 @@ module YouTubeStructs
property author_thumbnail : String property author_thumbnail : String
property banner : String? property banner : String?
property description_html : String property description_html : String
property paid : Bool
property total_views : Int64 property total_views : Int64
property sub_count : Int32 property sub_count : Int32
property joined : Time property joined : Time

8
src/invidious/data_structs/youtube/renderers/playlist_renderer.cr
View file

@ -37,12 +37,12 @@ module YouTubeStructs
json.array do json.array do
self.videos.each do |video| self.videos.each do |video|
json.object do json.object do
json.field "title", video.title json.field "title", video[:title]
json.field "videoId", video.id json.field "videoId", video[:id]
json.field "lengthSeconds", video.length_seconds json.field "lengthSeconds", video[:length_seconds]
json.field "videoThumbnails" do json.field "videoThumbnails" do
generate_thumbnails(json, video.id) generate_thumbnails(json, video[:id])
end end
end end
end end

2
src/invidious/data_structs/youtube/renderers/video_renderer.cr
View file

@ -20,7 +20,6 @@ module YouTubeStructs
property description_html : String property description_html : String
property length_seconds : Int32 property length_seconds : Int32
property live_now : Bool property live_now : Bool
property paid : Bool
property premium : Bool property premium : Bool
property premiere_timestamp : Time? property premiere_timestamp : Time?
@ -101,7 +100,6 @@ module YouTubeStructs
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale)) json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now json.field "liveNow", self.live_now
json.field "paid", self.paid
json.field "premium", self.premium json.field "premium", self.premium
json.field "isUpcoming", self.is_upcoming json.field "isUpcoming", self.is_upcoming

46
src/invidious/data_structs/youtube/videos.cr
View file

@ -1,10 +1,31 @@
module YouTubeStructs module YouTubeStructs
# Converter to serialize first level JSON data as methods for the videos struct
module VideoJSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
end
end
# Represents an watchable video in Invidious
#
# The video struct only takes three parameters:
# - ID: The video ID
#
# - Info:
# YT Video information (streams, captions, tiles, etc). This is then serialized
# into individual properties that either stores top level stuff or accesses
# further nested data.
#
# - Updated:
# A record of when the specific struct was created and inserted
# into the DB. This is then used to measure when to cache (or update)
# videos within the database.
struct Video struct Video
include DB::Serializable include DB::Serializable
property id : String property id : String
@[DB::Field(converter: Video::JSONConverter)] @[DB::Field(converter: YouTubeStructs::VideoJSONConverter)]
property info : Hash(String, JSON::Any) property info : Hash(String, JSON::Any)
property updated : Time property updated : Time
@ -20,12 +41,6 @@ module YouTubeStructs
@[DB::Field(ignore: true)] @[DB::Field(ignore: true)]
property description : String? property description : String?
module JSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
end
end
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
json.object do json.object do
json.field "type", "video" json.field "type", "video"
@ -277,10 +292,6 @@ module YouTubeStructs
info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d")) info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end end
def cookie
info["cookie"]?.try &.as_h.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
end
def allow_ratings def allow_ratings
r = info["videoDetails"]["allowRatings"]?.try &.as_bool r = info["videoDetails"]["allowRatings"]?.try &.as_bool
r.nil? ? false : r r.nil? ? false : r
@ -516,8 +527,13 @@ module YouTubeStructs
info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false
end end
def is_vr : Bool def is_vr : Bool?
info["streamingData"]?.try &.["adaptiveFormats"].as_a[0]?.try &.["projectionType"].as_s == "MESH" ? true : false || false projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
end
def projection_type : String?
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
end end
def wilson_score : Float64 def wilson_score : Float64
@ -531,9 +547,5 @@ module YouTubeStructs
def reason : String? def reason : String?
info["reason"]?.try &.as_s info["reason"]?.try &.as_s
end end
def session_token : String?
info["sessionToken"]?.try &.as_s?
end
end end
end end

30
src/invidious/helpers/extractors.cr
View file

@ -25,7 +25,7 @@ record AuthorFallback, name : String, id : String
# data is passed to the private `#parse()` method which returns a datastruct of the given # data is passed to the private `#parse()` method which returns a datastruct of the given
# type. Otherwise, nil is returned. # type. Otherwise, nil is returned.
private module Parsers private module Parsers
# Parses a InnerTube videoRenderer into a SearchVideo. Returns nil when the given object isn't a videoRenderer # Parses a InnerTube videoRenderer into a YouTubeStructs::VideoRenderer. Returns nil when the given object isn't a videoRenderer
# #
# A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not** # A videoRenderer renders a video to click on within the YouTube and Invidious UI. It is **not**
# the watchable video itself. # the watchable video itself.
@ -115,7 +115,7 @@ private module Parsers
end end
end end
# Parses a InnerTube channelRenderer into a SearchChannel. Returns nil when the given object isn't a channelRenderer # Parses a InnerTube channelRenderer into a YouTubeStructs::ChannelRenderer. Returns nil when the given object isn't a channelRenderer
# #
# A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not** # A channelRenderer renders a channel to click on within the YouTube and Invidious UI. It is **not**
# the channel page itself. # the channel page itself.
@ -161,7 +161,7 @@ private module Parsers
end end
end end
# Parses a InnerTube gridPlaylistRenderer into a SearchPlaylist. Returns nil when the given object isn't a gridPlaylistRenderer # Parses a InnerTube gridPlaylistRenderer into a YouTubeStructs::PlaylistRenderer. Returns nil when the given object isn't a gridPlaylistRenderer
# #
# A gridPlaylistRenderer renders a playlist, that is located in a grid, to click on within the YouTube and Invidious UI. # A gridPlaylistRenderer renders a playlist, that is located in a grid, to click on within the YouTube and Invidious UI.
# It is **not** the playlist itself. # It is **not** the playlist itself.
@ -196,7 +196,7 @@ private module Parsers
end end
end end
# Parses a InnerTube playlistRenderer into a SearchPlaylist. Returns nil when the given object isn't a playlistRenderer # Parses a InnerTube playlistRenderer into a YouTubeStructs::PlaylistRenderer. Returns nil when the given object isn't a playlistRenderer
# #
# A playlistRenderer renders a playlist to click on within the YouTube and Invidious UI. It is **not** the playlist itself. # A playlistRenderer renders a playlist to click on within the YouTube and Invidious UI. It is **not** the playlist itself.
# #
@ -536,7 +536,7 @@ def extract_item(item : JSON::Any, author_fallback : String? = "",
end end
# Parses multiple items from YouTube's initial JSON response into a more usable structure. # Parses multiple items from YouTube's initial JSON response into a more usable structure.
# The end result is an array of SearchItem. # The end result is an array of YouTubeStructs::Renderer.
def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil,
author_id_fallback : String? = nil) : Array(YouTubeStructs::Renderer) author_id_fallback : String? = nil) : Array(YouTubeStructs::Renderer)
items = [] of YouTubeStructs::Renderer items = [] of YouTubeStructs::Renderer
@ -565,27 +565,19 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
return items return items
end end
# Flattens all items from extracted items into a one dimensional array # Extracts videos (videoRenderer) from initial InnerTube response.
def flatten_items(items, target = nil) def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
if target.nil? extracted = extract_items(initial_data, author_fallback, author_id_fallback)
target = [] of YouTubeStructs::Renderer
end
items.each do |i| target = [] of YouTubeStructs::Renderer
extracted.each do |i|
if i.is_a?(YouTubeStructs::Category) if i.is_a?(YouTubeStructs::Category)
target = target += i.extract_renderers target += i.extract_renderers
else else
target << i target << i
end end
end end
return target
end
# Extracts videos (videoRenderer) from initial InnerTube response.
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
target = flatten_items(extracted)
return target.select(&.is_a?(YouTubeStructs::VideoRenderer)).map(&.as(YouTubeStructs::VideoRenderer)) return target.select(&.is_a?(YouTubeStructs::VideoRenderer)).map(&.as(YouTubeStructs::VideoRenderer))
end end

9
src/invidious/helpers/helpers.cr
View file

@ -15,13 +15,6 @@ struct SessionId
property issued : String property issued : String
end end
struct Annotation
include DB::Serializable
property id : String
property annotations : String
end
struct ConfigPreferences struct ConfigPreferences
include YAML::Serializable include YAML::Serializable
@ -444,7 +437,7 @@ def create_notification_stream(env, topics, connection_channel)
case topic case topic
when .match(/UC[A-Za-z0-9_-]{22}/) when .match(/UC[A-Za-z0-9_-]{22}/)
PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15", PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid = $1 AND published > $2 ORDER BY published DESC LIMIT 15",
topic, Time.unix(since.not_nil!), as: ChannelVideo).each do |video| topic, Time.unix(since.not_nil!), as: InvidiousStructs::ChannelVideo).each do |video|
response = JSON.parse(video.to_json(locale)) response = JSON.parse(video.to_json(locale))
if fields_text = env.params.query["fields"]? if fields_text = env.params.query["fields"]?

256
src/invidious/helpers/serialized_yt_data.cr
View file

@ -1,256 +0,0 @@
struct SearchVideo
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property published : Time
property views : Int64
property description_html : String
property length_seconds : Int32
property live_now : Bool
property premium : Bool
property premiere_timestamp : Time?
def to_xml(auto_generated, query_params, xml : XML::Builder)
query_params["v"] = self.id
xml.element("entry") do
xml.element("id") { xml.text "yt:video:#{self.id}" }
xml.element("yt:videoId") { xml.text self.id }
xml.element("yt:channelId") { xml.text self.ucid }
xml.element("title") { xml.text self.title }
xml.element("link", rel: "alternate", href: "#{HOST_URL}/watch?#{query_params}")
xml.element("author") do
if auto_generated
xml.element("name") { xml.text self.author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{self.ucid}" }
else
xml.element("name") { xml.text author }
xml.element("uri") { xml.text "#{HOST_URL}/channel/#{ucid}" }
end
end
xml.element("content", type: "xhtml") do
xml.element("div", xmlns: "http://www.w3.org/1999/xhtml") do
xml.element("a", href: "#{HOST_URL}/watch?#{query_params}") do
xml.element("img", src: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg")
end
xml.element("p", style: "word-break:break-word;white-space:pre-wrap") { xml.text html_to_content(self.description_html) }
end
end
xml.element("published") { xml.text self.published.to_s("%Y-%m-%dT%H:%M:%S%:z") }
xml.element("media:group") do
xml.element("media:title") { xml.text self.title }
xml.element("media:thumbnail", url: "#{HOST_URL}/vi/#{self.id}/mqdefault.jpg",
width: "320", height: "180")
xml.element("media:description") { xml.text html_to_content(self.description_html) }
end
xml.element("media:community") do
xml.element("media:statistics", views: self.views)
end
end
end
def to_xml(auto_generated, query_params, xml : XML::Builder | Nil = nil)
if xml
to_xml(HOST_URL, auto_generated, query_params, xml)
else
XML.build do |json|
to_xml(HOST_URL, auto_generated, query_params, xml)
end
end
end
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "viewCount", self.views
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "lengthSeconds", self.length_seconds
json.field "liveNow", self.live_now
json.field "premium", self.premium
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def is_upcoming
premiere_timestamp ? true : false
end
end
struct SearchPlaylistVideo
include DB::Serializable
property title : String
property id : String
property length_seconds : Int32
end
struct SearchPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property ucid : String
property video_count : Int32
property videos : Array(SearchPlaylistVideo)
property thumbnail : String?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoCount", self.video_count
json.field "videos" do
json.array do
self.videos.each do |video|
json.object do
json.field "title", video.title
json.field "videoId", video.id
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
generate_thumbnails(json, video.id)
end
end
end
end
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
struct SearchChannel
include DB::Serializable
property author : String
property ucid : String
property author_thumbnail : String
property subscriber_count : Int32
property video_count : Int32
property description_html : String
property auto_generated : Bool
def to_json(locale, json : JSON::Builder)
json.object do
json.field "type", "channel"
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "autoGenerated", self.auto_generated
json.field "subCount", self.subscriber_count
json.field "videoCount", self.video_count
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
class Category
include DB::Serializable
property title : String
property contents : Array(SearchItem) | Array(Video)
property url : String?
property description_html : String
property badges : Array(Tuple(String, String))?
def to_json(locale, json : JSON::Builder)
json.object do
json.field "title", self.title
json.field "contents", self.contents
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
end
alias SearchItem = SearchVideo | SearchChannel | SearchPlaylist | Category

4
src/invidious/jobs/pull_popular_videos_job.cr
View file

@ -6,7 +6,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40) GROUP BY channel ORDER BY COUNT(channel) DESC LIMIT 40)
ORDER BY ucid, published DESC ORDER BY ucid, published DESC
SQL SQL
POPULAR_VIDEOS = Atomic.new([] of ChannelVideo) POPULAR_VIDEOS = Atomic.new([] of InvidiousStructs::ChannelVideo)
private getter db : DB::Database private getter db : DB::Database
def initialize(@db) def initialize(@db)
@ -14,7 +14,7 @@ class Invidious::Jobs::PullPopularVideosJob < Invidious::Jobs::BaseJob
def begin def begin
loop do loop do
videos = db.query_all(QUERY, as: ChannelVideo) videos = db.query_all(QUERY, as: InvidiousStructs::ChannelVideo)
.sort_by(&.published) .sort_by(&.published)
.reverse .reverse

2
src/invidious/jobs/refresh_feeds_job.cr
View file

@ -26,7 +26,7 @@ class Invidious::Jobs::RefreshFeedsJob < Invidious::Jobs::BaseJob
begin begin
# Drop outdated views # Drop outdated views
column_array = get_column_array(db, view_name) column_array = get_column_array(db, view_name)
ChannelVideo.type_array.each_with_index do |name, i| InvidiousStructs::ChannelVideo.type_array.each_with_index do |name, i|
if name != column_array[i]? if name != column_array[i]?
LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}") LOGGER.info("RefreshFeedsJob: DROP MATERIALIZED VIEW #{view_name}")
db.exec("DROP MATERIALIZED VIEW #{view_name}") db.exec("DROP MATERIALIZED VIEW #{view_name}")

185
src/invidious/playlists.cr
View file

@ -1,4 +1,4 @@
struct PlaylistVideo struct YouTubeStructs::PlaylistVideo
include DB::Serializable include DB::Serializable
property title : String property title : String
@ -92,181 +92,16 @@ struct PlaylistVideo
end end
end end
struct Playlist
include DB::Serializable
property title : String
property id : String
property author : String
property author_thumbnail : String
property ucid : String
property description : String
property description_html : String
property video_count : Int32
property views : Int64
property updated : Time
property thumbnail : String?
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "playlist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "playlistThumbnail", self.thumbnail
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.not_nil!.gsub(/=\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, json)
end
end
end
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, json, continuation: continuation)
end
end
end
def privacy
PlaylistPrivacy::Public
end
end
enum PlaylistPrivacy enum PlaylistPrivacy
Public = 0 Public = 0
Unlisted = 1 Unlisted = 1
Private = 2 Private = 2
end end
struct InvidiousPlaylist
include DB::Serializable
property title : String
property id : String
property author : String
property description : String = ""
property video_count : Int32
property created : Time
property updated : Time
@[DB::Field(converter: InvidiousPlaylist::PlaylistPrivacyConverter)]
property privacy : PlaylistPrivacy = PlaylistPrivacy::Private
property index : Array(Int64)
@[DB::Field(ignore: true)]
property thumbnail_id : String?
module PlaylistPrivacyConverter
def self.from_rs(rs)
return PlaylistPrivacy.parse(String.new(rs.read(Slice(UInt8))))
end
end
def to_json(offset, locale, json : JSON::Builder, continuation : String? = nil)
json.object do
json.field "type", "invidiousPlaylist"
json.field "title", self.title
json.field "playlistId", self.id
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", nil
json.field "authorThumbnails", [] of String
json.field "description", html_to_content(self.description_html)
json.field "descriptionHtml", self.description_html
json.field "videoCount", self.video_count
json.field "viewCount", self.views
json.field "updated", self.updated.to_unix
json.field "isListed", self.privacy.public?
json.field "videos" do
json.array do
if !offset || offset == 0
index = PG_DB.query_one?("SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1", self.id, continuation, as: Int64)
offset = self.index.index(index) || 0
end
videos = get_playlist_videos(PG_DB, self, offset: offset, locale: locale, continuation: continuation)
videos.each_with_index do |video, index|
video.to_json(locale, json, offset + index)
end
end
end
end
end
def to_json(offset, locale, json : JSON::Builder? = nil, continuation : String? = nil)
if json
to_json(offset, locale, json, continuation: continuation)
else
JSON.build do |json|
to_json(offset, locale, json, continuation: continuation)
end
end
end
def thumbnail
@thumbnail_id ||= PG_DB.query_one?("SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1", self.id, self.index, as: String) || "-----------"
"/vi/#{@thumbnail_id}/mqdefault.jpg"
end
def author_thumbnail
nil
end
def ucid
nil
end
def views
0_i64
end
def description_html
HTML.escape(self.description).gsub("\n", "<br>")
end
end
def create_playlist(db, title, privacy, user) def create_playlist(db, title, privacy, user)
plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}" plid = "IVPL#{Random::Secure.urlsafe_base64(24)[0, 31]}"
playlist = InvidiousPlaylist.new({ playlist = InvidiousStructs::Playlist.new({
title: title.byte_slice(0, 150), title: title.byte_slice(0, 150),
id: plid, id: plid,
author: user.email, author: user.email,
@ -287,7 +122,7 @@ def create_playlist(db, title, privacy, user)
end end
def subscribe_playlist(db, user, playlist) def subscribe_playlist(db, user, playlist)
playlist = InvidiousPlaylist.new({ playlist = InvidiousStructs::Playlist.new({
title: playlist.title.byte_slice(0, 150), title: playlist.title.byte_slice(0, 150),
id: playlist.id, id: playlist.id,
author: user.email, author: user.email,
@ -346,7 +181,7 @@ end
def get_playlist(db, plid, locale, refresh = true, force_refresh = false) def get_playlist(db, plid, locale, refresh = true, force_refresh = false)
if plid.starts_with? "IV" if plid.starts_with? "IV"
if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if playlist = db.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
return playlist return playlist
else else
raise InfoException.new("Playlist does not exist.") raise InfoException.new("Playlist does not exist.")
@ -411,7 +246,7 @@ def fetch_playlist(plid, locale)
ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || "" ucid = author_info["title"]["runs"][0]["navigationEndpoint"]["browseEndpoint"]["browseId"]?.try &.as_s || ""
end end
return Playlist.new({ return YouTubeStructs::Playlist.new({
title: title, title: title,
id: plid, id: plid,
author: author, author: author,
@ -430,12 +265,12 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
# Show empy playlist if requested page is out of range # Show empy playlist if requested page is out of range
# (e.g, when a new playlist has been created, offset will be negative) # (e.g, when a new playlist has been created, offset will be negative)
if offset >= playlist.video_count || offset < 0 if offset >= playlist.video_count || offset < 0
return [] of PlaylistVideo return [] of YouTubeStructs::PlaylistVideo
end end
if playlist.is_a? InvidiousPlaylist if playlist.is_a? InvidiousStructs::Playlist
db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3", db.query_all("SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3",
playlist.id, playlist.index, offset, as: PlaylistVideo) playlist.id, playlist.index, offset, as: YouTubeStructs::PlaylistVideo)
else else
if offset >= 100 if offset >= 100
# Normalize offset to match youtube's behavior (100 videos chunck per request) # Normalize offset to match youtube's behavior (100 videos chunck per request)
@ -452,7 +287,7 @@ def get_playlist_videos(db, playlist, offset, locale = nil, continuation = nil)
end end
def extract_playlist_videos(initial_data : Hash(String, JSON::Any)) def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
videos = [] of PlaylistVideo videos = [] of YouTubeStructs::PlaylistVideo
if initial_data["contents"]? if initial_data["contents"]?
tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"] tabs = initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]
@ -493,7 +328,7 @@ def extract_playlist_videos(initial_data : Hash(String, JSON::Any))
length_seconds = 0 length_seconds = 0
end end
videos << PlaylistVideo.new({ videos << YouTubeStructs::PlaylistVideo.new({
title: title, title: title,
id: video_id, id: video_id,
author: author, author: author,

14
src/invidious/routes/api/v1/authenticated.cr
View file

@ -78,7 +78,7 @@ module Invidious::Routes::API::V1::Authenticated
values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}" values = "VALUES #{user.subscriptions.map { |id| %(('#{id}')) }.join(",")}"
end end
subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousChannel) subscriptions = PG_DB.query_all("SELECT * FROM channels WHERE id = ANY(#{values})", as: InvidiousStructs::Channel)
JSON.build do |json| JSON.build do |json|
json.array do json.array do
@ -127,7 +127,7 @@ module Invidious::Routes::API::V1::Authenticated
env.response.content_type = "application/json" env.response.content_type = "application/json"
user = env.get("user").as(User) user = env.get("user").as(User)
playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousPlaylist) playlists = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1", user.email, as: InvidiousStructs::Playlist)
JSON.build do |json| JSON.build do |json|
json.array do json.array do
@ -174,7 +174,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"] plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return error_json(404, "Playlist does not exist.")
end end
@ -207,7 +207,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"] plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return error_json(404, "Playlist does not exist.")
end end
@ -230,7 +230,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"] plid = env.params.url["plid"]
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return error_json(404, "Playlist does not exist.")
end end
@ -254,7 +254,7 @@ module Invidious::Routes::API::V1::Authenticated
return error_json(500, ex) return error_json(500, ex)
end end
playlist_video = PlaylistVideo.new({ playlist_video = YouTubeStructs::PlaylistVideo.new({
title: video.title, title: video.title,
id: video.id, id: video.id,
author: video.author, author: video.author,
@ -286,7 +286,7 @@ module Invidious::Routes::API::V1::Authenticated
plid = env.params.url["plid"] plid = env.params.url["plid"]
index = env.params.url["index"].to_i64(16) index = env.params.url["index"].to_i64(16)
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email && playlist.privacy.private? if !playlist || playlist.author != user.email && playlist.privacy.private?
return error_json(404, "Playlist does not exist.") return error_json(404, "Playlist does not exist.")
end end

4
src/invidious/routes/api/v1/channels.cr
View file

@ -19,7 +19,7 @@ module Invidious::Routes::API::V1::Channels
page = 1 page = 1
if channel.auto_generated if channel.auto_generated
videos = [] of SearchVideo videos = [] of YouTubeStructs::VideoRenderer
count = 0 count = 0
else else
begin begin
@ -208,7 +208,7 @@ module Invidious::Routes::API::V1::Channels
json.field "playlists" do json.field "playlists" do
json.array do json.array do
items.each do |item| items.each do |item|
item.to_json(locale, json) if item.is_a?(SearchPlaylist) item.to_json(locale, json) if item.is_a?(YouTubeStructs::PlaylistRenderer)
end end
end end
end end

2
src/invidious/routes/api/v1/videos.cr
View file

@ -239,7 +239,7 @@ module Invidious::Routes::API::V1::Videos
case source case source
when "archive" when "archive"
if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: Annotation)) if CONFIG.cache_annotations && (cached_annotation = PG_DB.query_one?("SELECT * FROM annotations WHERE id = $1", id, as: YouTubeStructs::Annotation))
annotations = cached_annotation.annotations annotations = cached_annotation.annotations
else else
index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0') index = CHARS_SAFE.index(id[0]).not_nil!.to_s.rjust(2, '0')

4
src/invidious/routes/channels.cr
View file

@ -29,7 +29,7 @@ module Invidious::Routes::Channels
item.author item.author
end end
end end
items = items.select(&.is_a?(SearchPlaylist)).map(&.as(SearchPlaylist)) items = items.select(&.is_a?(YouTubeStructs::PlaylistRenderer)).map(&.as(YouTubeStructs::PlaylistRenderer))
items.each { |item| item.author = "" } items.each { |item| item.author = "" }
else else
sort_options = {"newest", "oldest", "popular"} sort_options = {"newest", "oldest", "popular"}
@ -57,7 +57,7 @@ module Invidious::Routes::Channels
end end
items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by) items, continuation = fetch_channel_playlists(channel.ucid, channel.author, continuation, sort_by)
items = items.select { |item| item.is_a?(SearchPlaylist) }.map { |item| item.as(SearchPlaylist) } items = items.select { |item| item.is_a?(YouTubeStructs::PlaylistRenderer) }.map { |item| item.as(YouTubeStructs::PlaylistRenderer) }
items.each { |item| item.author = "" } items.each { |item| item.author = "" }
templated "playlists" templated "playlists"

10
src/invidious/routes/feeds.cr
View file

@ -15,13 +15,13 @@ module Invidious::Routes::Feeds
user = user.as(User) user = user.as(User)
items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) items_created = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist)
items_created.map! do |item| items_created.map! do |item|
item.author = "" item.author = ""
item item
end end
items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousPlaylist) items_saved = PG_DB.query_all("SELECT * FROM playlists WHERE author = $1 AND id NOT LIKE 'IV%' ORDER BY created", user.email, as: InvidiousStructs::Playlist)
items_saved.map! do |item| items_saved.map! do |item|
item.author = "" item.author = ""
item item
@ -169,7 +169,7 @@ module Invidious::Routes::Feeds
description_html = entry.xpath_node("group/description").not_nil!.to_s description_html = entry.xpath_node("group/description").not_nil!.to_s
views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64 views = entry.xpath_node("group/community/statistics").not_nil!.["views"].to_i64
SearchVideo.new({ YouTubeStructs::VideoRenderer.new({
title: title, title: title,
id: video_id, id: video_id,
author: author, author: author,
@ -264,7 +264,7 @@ module Invidious::Routes::Feeds
path = env.request.path path = env.request.path
if plid.starts_with? "IV" if plid.starts_with? "IV"
if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) if playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale) videos = get_playlist_videos(PG_DB, playlist, offset: 0, locale: locale)
return XML.build(indent: " ", encoding: "UTF-8") do |xml| return XML.build(indent: " ", encoding: "UTF-8") do |xml|
@ -405,7 +405,7 @@ module Invidious::Routes::Feeds
}.to_json }.to_json
PG_DB.exec("NOTIFY notifications, E'#{payload}'") PG_DB.exec("NOTIFY notifications, E'#{payload}'")
video = ChannelVideo.new({ video = InvidiousStructs::ChannelVideo.new({
id: id, id: id,
title: video.title, title: video.title,
published: published, published: published,

22
src/invidious/routes/playlists.cr
View file

@ -85,7 +85,7 @@ module Invidious::Routes::Playlists
sid = sid.as(String) sid = sid.as(String)
plid = env.params.query["list"]? plid = env.params.query["list"]?
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email if !playlist || playlist.author != user.email
return env.redirect referer return env.redirect referer
end end
@ -117,7 +117,7 @@ module Invidious::Routes::Playlists
return error_template(400, ex) return error_template(400, ex)
end end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email if !playlist || playlist.author != user.email
return env.redirect referer return env.redirect referer
end end
@ -149,7 +149,7 @@ module Invidious::Routes::Playlists
page ||= 1 page ||= 1
begin begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email if !playlist || playlist.author != user.email
return env.redirect referer return env.redirect referer
end end
@ -160,7 +160,7 @@ module Invidious::Routes::Playlists
begin begin
videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale) videos = get_playlist_videos(PG_DB, playlist, offset: (page - 1) * 100, locale: locale)
rescue ex rescue ex
videos = [] of PlaylistVideo videos = [] of YouTubeStructs::PlaylistVideo
end end
csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB) csrf_token = generate_response(sid, {":edit_playlist"}, HMAC_KEY, PG_DB)
@ -190,7 +190,7 @@ module Invidious::Routes::Playlists
return error_template(400, ex) return error_template(400, ex)
end end
playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one?("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email if !playlist || playlist.author != user.email
return env.redirect referer return env.redirect referer
end end
@ -233,7 +233,7 @@ module Invidious::Routes::Playlists
page ||= 1 page ||= 1
begin begin
playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousPlaylist) playlist = PG_DB.query_one("SELECT * FROM playlists WHERE id = $1", plid, as: InvidiousStructs::Playlist)
if !playlist || playlist.author != user.email if !playlist || playlist.author != user.email
return env.redirect referer return env.redirect referer
end end
@ -245,13 +245,13 @@ module Invidious::Routes::Playlists
if query if query
begin begin
search_query, count, items, operators = process_search_query(query, page, user, region: nil) search_query, count, items, operators = process_search_query(query, page, user, region: nil)
videos = items.select { |item| item.is_a? SearchVideo }.map { |item| item.as(SearchVideo) } videos = items.select { |item| item.is_a? YouTubeStructs::VideoRenderer }.map { |item| item.as(YouTubeStructs::VideoRenderer) }
rescue ex rescue ex
videos = [] of SearchVideo videos = [] of YouTubeStructs::VideoRenderer
count = 0 count = 0
end end
else else
videos = [] of SearchVideo videos = [] of YouTubeStructs::VideoRenderer
count = 0 count = 0
end end
@ -311,7 +311,7 @@ module Invidious::Routes::Playlists
begin begin
playlist_id = env.params.query["playlist_id"] playlist_id = env.params.query["playlist_id"]
playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousPlaylist) playlist = get_playlist(PG_DB, playlist_id, locale).as(InvidiousStructs::Playlist)
raise "Invalid user" if playlist.author != user.email raise "Invalid user" if playlist.author != user.email
rescue ex rescue ex
if redirect if redirect
@ -351,7 +351,7 @@ module Invidious::Routes::Playlists
end end
end end
playlist_video = PlaylistVideo.new({ playlist_video = YouTubeStructs::PlaylistVideo.new({
title: video.title, title: video.title,
id: video.id, id: video.id,
author: video.author, author: video.author,

20
src/invidious/search.cr
View file

@ -17,9 +17,9 @@ def channel_search(query, page, channel)
continuationItems = response_json["onResponseReceivedActions"]? continuationItems = response_json["onResponseReceivedActions"]?
.try &.[0]["appendContinuationItemsAction"]["continuationItems"] .try &.[0]["appendContinuationItemsAction"]["continuationItems"]
return 0, [] of SearchItem if !continuationItems return 0, [] of YouTubeStructs::Renderer if !continuationItems
items = [] of SearchItem items = [] of YouTubeStructs::Renderer
continuationItems.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item| continuationItems.as_a.select(&.as_h.has_key?("itemSectionRenderer")).each { |item|
extract_item(item["itemSectionRenderer"]["contents"].as_a[0]) extract_item(item["itemSectionRenderer"]["contents"].as_a[0])
.try { |t| items << t } .try { |t| items << t }
@ -29,7 +29,7 @@ def channel_search(query, page, channel)
end end
def search(query, search_params = produce_search_params(content_type: "all"), region = nil) def search(query, search_params = produce_search_params(content_type: "all"), region = nil)
return 0, [] of SearchItem if query.empty? return 0, [] of YouTubeStructs::Renderer if query.empty?
client_config = YoutubeAPI::ClientConfig.new(region: region) client_config = YoutubeAPI::ClientConfig.new(region: region)
initial_data = YoutubeAPI.search(query, search_params, client_config: client_config) initial_data = YoutubeAPI.search(query, search_params, client_config: client_config)
@ -219,10 +219,10 @@ def process_search_query(query, page, user, region)
to_tsvector(#{view_name}.author) to_tsvector(#{view_name}.author)
as document as document
FROM #{view_name} FROM #{view_name}
) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo) ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: InvidiousStructs::ChannelVideo)
count = items.size count = items.size
else else
items = [] of ChannelVideo items = [] of InvidiousStructs::ChannelVideo
count = 0 count = 0
end end
else else
@ -234,14 +234,10 @@ def process_search_query(query, page, user, region)
# Light processing to flatten search results out of Categories. # Light processing to flatten search results out of Categories.
# They should ideally be supported in the future. # They should ideally be supported in the future.
items_without_category = [] of SearchItem | ChannelVideo items_without_category = [] of YouTubeStructs::Renderer | InvidiousStructs::ChannelVideo
items.each do |i| items.each do |i|
if i.is_a? Category if i.is_a? YouTubeStructs::Category
i.contents.each do |nest_i| items_without_category += i.extract_renderers
if !nest_i.is_a? Video
items_without_category << nest_i
end
end
else else
items_without_category << i items_without_category << i
end end

12
src/invidious/users.cr
View file

@ -503,8 +503,8 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
args = arg_array(notifications) args = arg_array(notifications)
notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: ChannelVideo) notifications = db.query_all("SELECT * FROM channel_videos WHERE id IN (#{args}) ORDER BY published DESC", args: notifications, as: InvidiousStructs::ChannelVideo)
videos = [] of ChannelVideo videos = [] of InvidiousStructs::ChannelVideo
notifications.sort_by! { |video| video.published }.reverse! notifications.sort_by! { |video| video.published }.reverse!
@ -530,11 +530,11 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
else else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end end
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: ChannelVideo) videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY ucid, published DESC", as: InvidiousStructs::ChannelVideo)
else else
# Show latest video from each channel # Show latest video from each channel
videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo) videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: InvidiousStructs::ChannelVideo)
end end
videos.sort_by! { |video| video.published }.reverse! videos.sort_by! { |video| video.published }.reverse!
@ -547,11 +547,11 @@ def get_subscription_feed(db, user, max_results = 40, page = 1)
else else
values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}" values = "VALUES #{user.watched.map { |id| %(('#{id}')) }.join(",")}"
end end
videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE NOT id = ANY (#{values}) ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: InvidiousStructs::ChannelVideo)
else else
# Sort subscriptions as normal # Sort subscriptions as normal
videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo) videos = PG_DB.query_all("SELECT * FROM #{view_name} ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: InvidiousStructs::ChannelVideo)
end end
end end

585
src/invidious/videos.cr
View file

@ -221,583 +221,6 @@ VIDEO_FORMATS = {
"397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"}, "397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
} }
struct VideoPreferences
include JSON::Serializable
property annotations : Bool
property autoplay : Bool
property comments : Array(String)
property continue : Bool
property continue_autoplay : Bool
property controls : Bool
property listen : Bool
property local : Bool
property preferred_captions : Array(String)
property player_style : String
property quality : String
property quality_dash : String
property raw : Bool
property region : String?
property related_videos : Bool
property speed : Float32 | Float64
property video_end : Float64 | Int32
property video_loop : Bool
property extend_desc : Bool
property video_start : Float64 | Int32
property volume : Int32
property vr_mode : Bool
end
struct Video
include DB::Serializable
property id : String
@[DB::Field(converter: Video::JSONConverter)]
property info : Hash(String, JSON::Any)
property updated : Time
@[DB::Field(ignore: true)]
property captions : Array(Caption)?
@[DB::Field(ignore: true)]
property adaptive_fmts : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property fmt_stream : Array(Hash(String, JSON::Any))?
@[DB::Field(ignore: true)]
property description : String?
module JSONConverter
def self.from_rs(rs)
JSON.parse(rs.read(String)).as_h
end
end
def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder)
json.object do
json.field "type", "video"
json.field "title", self.title
json.field "videoId", self.id
json.field "error", info["reason"] if info["reason"]?
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
end
json.field "storyboards" do
generate_storyboards(json, self.id, self.storyboards)
end
json.field "description", self.description
json.field "descriptionHtml", self.description_html
json.field "published", self.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(self.published, locale))
json.field "keywords", self.keywords
json.field "viewCount", self.views
json.field "likeCount", self.likes
json.field "dislikeCount", self.dislikes
json.field "paid", self.paid
json.field "premium", self.premium
json.field "isFamilyFriendly", self.is_family_friendly
json.field "allowedRegions", self.allowed_regions
json.field "genre", self.genre
json.field "genreUrl", self.genre_url
json.field "author", self.author
json.field "authorId", self.ucid
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", self.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "subCountText", self.sub_count_text
json.field "lengthSeconds", self.length_seconds
json.field "allowRatings", self.allow_ratings
json.field "rating", self.average_rating
json.field "isListed", self.is_listed
json.field "liveNow", self.live_now
json.field "isUpcoming", self.is_upcoming
if self.premiere_timestamp
json.field "premiereTimestamp", self.premiere_timestamp.try &.to_unix
end
if hlsvp = self.hls_manifest_url
hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
json.field "hlsUrl", hlsvp
end
json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{id}"
json.field "adaptiveFormats" do
json.array do
self.adaptive_fmts.each do |fmt|
json.object do
json.field "index", "#{fmt["indexRange"]["start"]}-#{fmt["indexRange"]["end"]}"
json.field "bitrate", fmt["bitrate"].as_i.to_s
json.field "init", "#{fmt["initRange"]["start"]}-#{fmt["initRange"]["end"]}"
json.field "url", fmt["url"]
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "clen", fmt["contentLength"]
json.field "lmt", fmt["lastModified"]
json.field "projectionType", fmt["projectionType"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
end
end
end
end
json.field "formatStreams" do
json.array do
self.fmt_stream.each do |fmt|
json.object do
json.field "url", fmt["url"]
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "quality", fmt["quality"]
fmt_info = itag_to_metadata?(fmt["itag"])
if fmt_info
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
end
end
end
end
json.field "captions" do
json.array do
self.captions.each do |caption|
json.object do
json.field "label", caption.name
json.field "languageCode", caption.languageCode
json.field "url", "/api/v1/captions/#{id}?label=#{URI.encode_www_form(caption.name)}"
end
end
end
end
json.field "recommendedVideos" do
json.array do
self.related_videos.each do |rv|
if rv["id"]?
json.object do
json.field "videoId", rv["id"]
json.field "title", rv["title"]
json.field "videoThumbnails" do
generate_thumbnails(json, rv["id"])
end
json.field "author", rv["author"]
json.field "authorUrl", rv["author_url"]?
json.field "authorId", rv["ucid"]?
if rv["author_thumbnail"]?
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", rv["author_thumbnail"]?.try &.gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
json.field "viewCountText", rv["short_view_count_text"]?
json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
end
end
end
end
end
end
end
def to_json(locale, json : JSON::Builder | Nil = nil)
if json
to_json(locale, json)
else
JSON.build do |json|
to_json(locale, json)
end
end
end
def title
info["videoDetails"]["title"]?.try &.as_s || ""
end
def ucid
info["videoDetails"]["channelId"]?.try &.as_s || ""
end
def author
info["videoDetails"]["author"]?.try &.as_s || ""
end
def length_seconds : Int32
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["lengthSeconds"]?.try &.as_s.to_i ||
info["videoDetails"]["lengthSeconds"]?.try &.as_s.to_i || 0
end
def views : Int64
info["videoDetails"]["viewCount"]?.try &.as_s.to_i64 || 0_i64
end
def likes : Int64
info["likes"]?.try &.as_i64 || 0_i64
end
def dislikes : Int64
info["dislikes"]?.try &.as_i64 || 0_i64
end
def average_rating : Float64
# (likes / (likes + dislikes) * 4 + 1)
info["videoDetails"]["averageRating"]?.try { |t| t.as_f? || t.as_i64?.try &.to_f64 }.try &.round(4) || 0.0
end
def published : Time
info["microformat"]?.try &.["playerMicroformatRenderer"]?.try &.["publishDate"]?.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
end
def published=(other : Time)
info["microformat"].as_h["playerMicroformatRenderer"].as_h["publishDate"] = JSON::Any.new(other.to_s("%Y-%m-%d"))
end
def allow_ratings
r = info["videoDetails"]["allowRatings"]?.try &.as_bool
r.nil? ? false : r
end
def live_now
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["liveBroadcastDetails"]?.try &.["isLiveNow"]?.try &.as_bool || false
end
def is_listed
info["videoDetails"]["isCrawlable"]?.try &.as_bool || false
end
def is_upcoming
info["videoDetails"]["isUpcoming"]?.try &.as_bool || false
end
def premiere_timestamp : Time?
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["liveBroadcastDetails"]?.try &.["startTimestamp"]?.try { |t| Time.parse_rfc3339(t.as_s) }
end
def keywords
info["videoDetails"]["keywords"]?.try &.as_a.map &.as_s || [] of String
end
def related_videos
info["relatedVideos"]?.try &.as_a.map { |h| h.as_h.transform_values &.as_s } || [] of Hash(String, String)
end
def allowed_regions
info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["availableCountries"]?.try &.as_a.map &.as_s || [] of String
end
def author_thumbnail : String
info["authorThumbnail"]?.try &.as_s || ""
end
def sub_count_text : String
info["subCountText"]?.try &.as_s || "-"
end
def fmt_stream
return @fmt_stream.as(Array(Hash(String, JSON::Any))) if @fmt_stream
fmt_stream = info["streamingData"]?.try &.["formats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
fmt_stream.each do |fmt|
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
s.each do |k, v|
fmt[k] = JSON::Any.new(v)
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
fmt["url"] = JSON::Any.new("#{fmt["url"]}&region=#{self.info["region"]}") if self.info["region"]?
end
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
@fmt_stream = fmt_stream
return @fmt_stream.as(Array(Hash(String, JSON::Any)))
end
def adaptive_fmts
return @adaptive_fmts.as(Array(Hash(String, JSON::Any))) if @adaptive_fmts
fmt_stream = info["streamingData"]?.try &.["adaptiveFormats"]?.try &.as_a.map &.as_h || [] of Hash(String, JSON::Any)
fmt_stream.each do |fmt|
if s = (fmt["cipher"]? || fmt["signatureCipher"]?).try { |h| HTTP::Params.parse(h.as_s) }
s.each do |k, v|
fmt[k] = JSON::Any.new(v)
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
end
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
fmt["url"] = JSON::Any.new("#{fmt["url"]}&region=#{self.info["region"]}") if self.info["region"]?
end
# See https://github.com/TeamNewPipe/NewPipe/issues/2415
# Some streams are segmented by URL `sq/` rather than index, for now we just filter them out
fmt_stream.reject! { |f| !f["indexRange"]? }
fmt_stream.sort_by! { |f| f["width"]?.try &.as_i || 0 }
@adaptive_fmts = fmt_stream
return @adaptive_fmts.as(Array(Hash(String, JSON::Any)))
end
def video_streams
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("video")
end
def audio_streams
adaptive_fmts.select &.["mimeType"]?.try &.as_s.starts_with?("audio")
end
def storyboards
storyboards = info["storyboards"]?
.try &.as_h
.try &.["playerStoryboardSpecRenderer"]?
.try &.["spec"]?
.try &.as_s.split("|")
if !storyboards
if storyboard = info["storyboards"]?
.try &.as_h
.try &.["playerLiveStoryboardSpecRenderer"]?
.try &.["spec"]?
.try &.as_s
return [{
url: storyboard.split("#")[0],
width: 106,
height: 60,
count: -1,
interval: 5000,
storyboard_width: 3,
storyboard_height: 3,
storyboard_count: -1,
}]
end
end
items = [] of NamedTuple(
url: String,
width: Int32,
height: Int32,
count: Int32,
interval: Int32,
storyboard_width: Int32,
storyboard_height: Int32,
storyboard_count: Int32)
return items if !storyboards
url = URI.parse(storyboards.shift)
params = HTTP::Params.parse(url.query || "")
storyboards.each_with_index do |storyboard, i|
width, height, count, storyboard_width, storyboard_height, interval, _, sigh = storyboard.split("#")
params["sigh"] = sigh
url.query = params.to_s
width = width.to_i
height = height.to_i
count = count.to_i
interval = interval.to_i
storyboard_width = storyboard_width.to_i
storyboard_height = storyboard_height.to_i
storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
items << {
url: url.to_s.sub("$L", i).sub("$N", "M$M"),
width: width,
height: height,
count: count,
interval: interval,
storyboard_width: storyboard_width,
storyboard_height: storyboard_height,
storyboard_count: storyboard_count,
}
end
items
end
def paid
reason = info["playabilityStatus"]?.try &.["reason"]?
paid = reason == "This video requires payment to watch." ? true : false
paid
end
def premium
keywords.includes? "YouTube Red"
end
def captions : Array(Caption)
return @captions.as(Array(Caption)) if @captions
captions = info["captions"]?.try &.["playerCaptionsTracklistRenderer"]?.try &.["captionTracks"]?.try &.as_a.map do |caption|
name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
languageCode = caption["languageCode"].to_s
baseUrl = caption["baseUrl"].to_s
caption = Caption.new(name.to_s, languageCode, baseUrl)
caption.name = caption.name.split(" - ")[0]
caption
end
captions ||= [] of Caption
@captions = captions
return @captions.as(Array(Caption))
end
def description
description = info["microformat"]?.try &.["playerMicroformatRenderer"]?
.try &.["description"]?.try &.["simpleText"]?.try &.as_s || ""
end
# TODO
def description=(value : String)
@description = value
end
def description_html
info["descriptionHtml"]?.try &.as_s || "<p></p>"
end
def description_html=(value : String)
info["descriptionHtml"] = JSON::Any.new(value)
end
def short_description
info["shortDescription"]?.try &.as_s? || ""
end
def hls_manifest_url : String?
info["streamingData"]?.try &.["hlsManifestUrl"]?.try &.as_s
end
def dash_manifest_url
info["streamingData"]?.try &.["dashManifestUrl"]?.try &.as_s
end
def genre : String
info["genre"]?.try &.as_s || ""
end
def genre_url : String?
info["genreUcid"]? ? "/channel/#{info["genreUcid"]}" : nil
end
def license : String?
info["license"]?.try &.as_s
end
def is_family_friendly : Bool
info["microformat"]?.try &.["playerMicroformatRenderer"]["isFamilySafe"]?.try &.as_bool || false
end
def is_vr : Bool?
projection_type = info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
return {"EQUIRECTANGULAR", "MESH"}.includes? projection_type
end
def projection_type : String?
return info.dig?("streamingData", "adaptiveFormats", 0, "projectionType").try &.as_s
end
def wilson_score : Float64
ci_lower_bound(likes, likes + dislikes).round(4)
end
def engagement : Float64
(((likes + dislikes) / views) * 100).round(4)
end
def reason : String?
info["reason"]?.try &.as_s
end
end
struct Caption
property name
property languageCode
property baseUrl
getter name : String
getter languageCode : String
getter baseUrl : String
setter name
def initialize(@name, @languageCode, @baseUrl)
end
end
class VideoRedirect < Exception class VideoRedirect < Exception
property video_id : String property video_id : String
@ -942,7 +365,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil, context_
end end
def get_video(id, db, refresh = true, region = nil, force_refresh = false) def get_video(id, db, refresh = true, region = nil, force_refresh = false)
if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: Video)) && !region if (video = db.query_one?("SELECT * FROM videos WHERE id = $1", id, as: YouTubeStructs::Video)) && !region
# If record was last updated over 10 minutes ago, or video has since premiered, # If record was last updated over 10 minutes ago, or video has since premiered,
# refresh (expire param in response lasts for 6 hours) # refresh (expire param in response lasts for 6 hours)
if (refresh && if (refresh &&
@ -967,6 +390,8 @@ def get_video(id, db, refresh = true, region = nil, force_refresh = false)
return video return video
end end
# TODO make private. All instances of fetching video should be done from get_video() to
# allow for caching.
def fetch_video(id, region) def fetch_video(id, region)
info = extract_video_info(video_id: id) info = extract_video_info(video_id: id)
@ -993,7 +418,7 @@ def fetch_video(id, region)
raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]? raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]?
video = Video.new({ video = YouTubeStructs::Video.new({
id: id, id: id,
info: info, info: info,
updated: Time.utc, updated: Time.utc,
@ -1116,7 +541,7 @@ def process_video_params(query, preferences)
controls ||= 1 controls ||= 1
controls = controls >= 1 controls = controls >= 1
params = VideoPreferences.new({ params = InvidiousStructs::VideoPreferences.new({
annotations: annotations, annotations: annotations,
autoplay: autoplay, autoplay: autoplay,
comments: comments, comments: comments,

8
src/invidious/views/components/item.ecr
View file

@ -1,7 +1,7 @@
<div class="pure-u-1 pure-u-md-1-4"> <div class="pure-u-1 pure-u-md-1-4">
<div class="h-box"> <div class="h-box">
<% case item when %> <% case item when %>
<% when SearchChannel %> <% when YouTubeStructs::ChannelRenderer %>
<a href="/channel/<%= item.ucid %>"> <a href="/channel/<%= item.ucid %>">
<% if !env.get("preferences").as(Preferences).thin_mode %> <% if !env.get("preferences").as(Preferences).thin_mode %>
<center> <center>
@ -13,7 +13,7 @@
<p><%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %></p> <p><%= translate(locale, "`x` subscribers", number_with_separator(item.subscriber_count)) %></p>
<% if !item.auto_generated %><p><%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %></p><% end %> <% if !item.auto_generated %><p><%= translate(locale, "`x` videos", number_with_separator(item.video_count)) %></p><% end %>
<h5><%= item.description_html %></h5> <h5><%= item.description_html %></h5>
<% when SearchPlaylist, InvidiousPlaylist %> <% when YouTubeStructs::PlaylistRenderer, InvidiousStructs::Playlist %>
<% if item.id.starts_with? "RD" %> <% if item.id.starts_with? "RD" %>
<% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %> <% url = "/mix?list=#{item.id}&continuation=#{URI.parse(item.thumbnail || "/vi/-----------").request_target.split("/")[2]}" %>
<% else %> <% else %>
@ -47,7 +47,7 @@
<a href="/channel/<%= item.ucid %>"> <a href="/channel/<%= item.ucid %>">
<p dir="auto"><b><%= HTML.escape(item.author) %></b></p> <p dir="auto"><b><%= HTML.escape(item.author) %></b></p>
</a> </a>
<% when PlaylistVideo %> <% when YouTubeStructs::PlaylistVideo %>
<a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>"> <a style="width:100%" href="/watch?v=<%= item.id %>&list=<%= item.plid %>">
<% if !env.get("preferences").as(Preferences).thin_mode %> <% if !env.get("preferences").as(Preferences).thin_mode %>
<div class="thumbnail"> <div class="thumbnail">
@ -109,7 +109,7 @@
</div> </div>
<% end %> <% end %>
</div> </div>
<% when Category %> <% when YouTubeStructs::Category %>
<% else %> <% else %>
<a style="width:100%" href="/watch?v=<%= item.id %>"> <a style="width:100%" href="/watch?v=<%= item.id %>">
<% if !env.get("preferences").as(Preferences).thin_mode %> <% if !env.get("preferences").as(Preferences).thin_mode %>

2
src/invidious/views/edit_playlist.ecr
View file

@ -44,7 +44,7 @@
<input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>"> <input type="hidden" name="csrf_token" value="<%= URI.encode_www_form(csrf_token) %>">
</form> </form>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> <% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<div class="h-box" style="text-align:right"> <div class="h-box" style="text-align:right">
<h3> <h3>
<a href="/add_playlist_items?list=<%= plid %>"><i class="icon ion-md-add"></i></a> <a href="/add_playlist_items?list=<%= plid %>"><i class="icon ion-md-add"></i></a>

12
src/invidious/views/playlist.ecr
View file

@ -9,7 +9,7 @@
<div class="pure-g h-box"> <div class="pure-g h-box">
<div class="pure-u-2-3"> <div class="pure-u-2-3">
<h3><%= title %></h3> <h3><%= title %></h3>
<% if playlist.is_a? InvidiousPlaylist %> <% if playlist.is_a? InvidiousStructs::Playlist %>
<b> <b>
<% if playlist.author == user.try &.email %> <% if playlist.author == user.try &.email %>
<a href="/feed/playlists"><%= author %></a> | <a href="/feed/playlists"><%= author %></a> |
@ -18,7 +18,7 @@
<% end %> <% end %>
<%= translate(locale, "`x` videos", "#{playlist.video_count}") %> | <%= translate(locale, "`x` videos", "#{playlist.video_count}") %> |
<%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> | <%= translate(locale, "Updated `x` ago", recode_date(playlist.updated, locale)) %> |
<% case playlist.as(InvidiousPlaylist).privacy when %> <% case playlist.as(InvidiousStructs::Playlist).privacy when %>
<% when PlaylistPrivacy::Public %> <% when PlaylistPrivacy::Public %>
<i class="icon ion-md-globe"></i> <%= translate(locale, "Public") %> <i class="icon ion-md-globe"></i> <%= translate(locale, "Public") %>
<% when PlaylistPrivacy::Unlisted %> <% when PlaylistPrivacy::Unlisted %>
@ -35,7 +35,7 @@
</b> </b>
<% end %> <% end %>
<% if !playlist.is_a? InvidiousPlaylist %> <% if !playlist.is_a? InvidiousStructs::Playlist %>
<div class="pure-u-2-3"> <div class="pure-u-2-3">
<a href="https://www.youtube.com/playlist?list=<%= playlist.id %>"> <a href="https://www.youtube.com/playlist?list=<%= playlist.id %>">
<%= translate(locale, "View playlist on YouTube") %> <%= translate(locale, "View playlist on YouTube") %>
@ -50,7 +50,7 @@
<div class="pure-u-1-3" style="text-align:right"> <div class="pure-u-1-3" style="text-align:right">
<h3> <h3>
<div class="pure-g user-field"> <div class="pure-g user-field">
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> <% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div> <div class="pure-u-1-3"><a href="/edit_playlist?list=<%= plid %>"><i class="icon ion-md-create"></i></a></div>
<div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div> <div class="pure-u-1-3"><a href="/delete_playlist?list=<%= plid %>"><i class="icon ion-md-trash"></i></a></div>
<% else %> <% else %>
@ -72,7 +72,7 @@
</div> </div>
</div> </div>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> <% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<div class="h-box" style="text-align:right"> <div class="h-box" style="text-align:right">
<h3> <h3>
<a href="/add_playlist_items?list=<%= plid %>"><i class="icon ion-md-add"></i></a> <a href="/add_playlist_items?list=<%= plid %>"><i class="icon ion-md-add"></i></a>
@ -84,7 +84,7 @@
<hr> <hr>
</div> </div>
<% if playlist.is_a?(InvidiousPlaylist) && playlist.author == user.try &.email %> <% if playlist.is_a?(InvidiousStructs::Playlist) && playlist.author == user.try &.email %>
<script id="playlist_data" type="application/json"> <script id="playlist_data" type="application/json">
<%= <%=
{ {