Cleanup videos (#3238)

pull/3408/head^2
Samantaz Fox 2 years ago
commit 516efd2df3
No known key found for this signature in database
GPG Key ID: F42821059186176E

@ -1 +1 @@
Subproject commit c401dd9203434b561022242c24b0c200d72284c0
Subproject commit dfd53ea6ceb3cbcbbce6004f6ce60b330ad0f9b1

@ -0,0 +1,168 @@
require "../../parsers_helper.cr"
Spectator.describe "parse_video_info" do
it "parses a regular video" do
# Enable mock
_player = load_mock("video/regular_mrbeast.player")
_next = load_mock("video/regular_mrbeast.next")
raw_data = _player.merge!(_next)
info = parse_video_info("2isYuQZMbdU", raw_data)
# Some basic verifications
expect(typeof(info)).to eq(Hash(String, JSON::Any))
expect(info["videoType"].as_s).to eq("Video")
# Basic video infos
expect(info["title"].as_s).to eq("I Gave My 100,000,000th Subscriber An Island")
expect(info["views"].as_i).to eq(32_846_329)
expect(info["likes"].as_i).to eq(2_611_650)
# For some reason the video length from VideoDetails and the
# one from microformat differs by 1s...
expect(info["lengthSeconds"].as_i).to be_between(930_i64, 931_i64)
expect(info["published"].as_s).to eq("2022-08-04T00:00:00Z")
# Extra video infos
expect(info["allowedRegions"].as_a).to_not be_empty
expect(info["allowedRegions"].as_a.size).to eq(249)
expect(info["allowedRegions"].as_a).to contain(
"AD", "BA", "BB", "BW", "BY", "EG", "GG", "HN", "NP", "NR", "TR",
"TT", "TV", "TW", "TZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU",
"WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"
)
expect(info["keywords"].as_a).to be_empty
expect(info["allowRatings"].as_bool).to be_true
expect(info["isFamilyFriendly"].as_bool).to be_true
expect(info["isListed"].as_bool).to be_true
expect(info["isUpcoming"].as_bool).to be_false
# Related videos
expect(info["relatedVideos"].as_a.size).to eq(19)
expect(info["relatedVideos"][0]["id"]).to eq("tVWWp1PqDus")
expect(info["relatedVideos"][0]["title"]).to eq("100 Girls Vs 100 Boys For $500,000")
expect(info["relatedVideos"][0]["author"]).to eq("MrBeast")
expect(info["relatedVideos"][0]["ucid"]).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA")
expect(info["relatedVideos"][0]["view_count"]).to eq("49702799")
expect(info["relatedVideos"][0]["short_view_count"]).to eq("49M")
expect(info["relatedVideos"][0]["author_verified"]).to eq("true")
# Description
description = "🚀Launch a store on Shopify, Ill buy from 100 random stores that do ▸ "
expect(info["description"].as_s).to start_with(description)
expect(info["shortDescription"].as_s).to start_with(description)
expect(info["descriptionHtml"].as_s).to start_with(description)
# Video metadata
expect(info["genre"].as_s).to eq("Entertainment")
expect(info["genreUcid"].as_s).to be_empty
expect(info["license"].as_s).to be_empty
# Author infos
expect(info["author"].as_s).to eq("MrBeast")
expect(info["ucid"].as_s).to eq("UCX6OQ3DkcsbYNE6H8uQQuVA")
expect(info["authorThumbnail"].as_s).to eq(
"https://yt3.ggpht.com/ytc/AMLnZu84dsnlYtuUFBMC8imQs0IUcTKA9khWAmUOgQZltw=s48-c-k-c0x00ffffff-no-rj"
)
expect(info["authorVerified"].as_bool).to be_true
expect(info["subCountText"].as_s).to eq("101M")
end
it "parses a regular video with no descrition/comments" do
# Enable mock
_player = load_mock("video/regular_no-description.player")
_next = load_mock("video/regular_no-description.next")
raw_data = _player.merge!(_next)
info = parse_video_info("iuevw6218F0", raw_data)
# Some basic verifications
expect(typeof(info)).to eq(Hash(String, JSON::Any))
expect(info["videoType"].as_s).to eq("Video")
# Basic video infos
expect(info["title"].as_s).to eq("Chris Rea - Auberge")
expect(info["views"].as_i).to eq(10_356_197)
expect(info["likes"].as_i).to eq(0)
expect(info["lengthSeconds"].as_i).to eq(283_i64)
expect(info["published"].as_s).to eq("2012-05-21T00:00:00Z")
# Extra video infos
expect(info["allowedRegions"].as_a).to_not be_empty
expect(info["allowedRegions"].as_a.size).to eq(249)
expect(info["allowedRegions"].as_a).to contain(
"AD", "BA", "BB", "BW", "BY", "EG", "GG", "HN", "NP", "NR", "TR",
"TT", "TV", "TW", "TZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU",
"WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"
)
expect(info["keywords"].as_a).to_not be_empty
expect(info["keywords"].as_a.size).to eq(4)
expect(info["keywords"].as_a).to contain_exactly(
"Chris",
"Rea",
"Auberge",
"1991"
).in_any_order
expect(info["allowRatings"].as_bool).to be_true
expect(info["isFamilyFriendly"].as_bool).to be_true
expect(info["isListed"].as_bool).to be_true
expect(info["isUpcoming"].as_bool).to be_false
# Related videos
expect(info["relatedVideos"].as_a.size).to eq(19)
expect(info["relatedVideos"][0]["id"]).to eq("0bkrY_V0yZg")
expect(info["relatedVideos"][0]["title"]).to eq(
"Chris Rea Best Songs Collection - Chris Rea Greatest Hits Full Album 2022"
)
expect(info["relatedVideos"][0]["author"]).to eq("Rock Ultimate")
expect(info["relatedVideos"][0]["ucid"]).to eq("UCekSc2A19di9koUIpj8gxlQ")
expect(info["relatedVideos"][0]["view_count"]).to eq("1992412")
expect(info["relatedVideos"][0]["short_view_count"]).to eq("1.9M")
expect(info["relatedVideos"][0]["author_verified"]).to eq("false")
# Description
expect(info["description"].as_s).to eq(" ")
expect(info["shortDescription"].as_s).to be_empty
expect(info["descriptionHtml"].as_s).to eq("<p></p>")
# Video metadata
expect(info["genre"].as_s).to eq("Music")
expect(info["genreUcid"].as_s).to be_empty
expect(info["license"].as_s).to be_empty
# Author infos
expect(info["author"].as_s).to eq("ChrisReaOfficial")
expect(info["ucid"].as_s).to eq("UC_5q6nWPbD30-y6oiWF_oNA")
expect(info["authorThumbnail"].as_s).to be_empty
expect(info["authorVerified"].as_bool).to be_false
expect(info["subCountText"].as_s).to eq("-")
end
end

@ -1,6 +1,6 @@
require "../../parsers_helper.cr"
Spectator.describe Invidious::Hashtag do
Spectator.describe "parse_video_info" do
it "parses scheduled livestreams data (test 1)" do
# Enable mock
_player = load_mock("video/scheduled_live_nintendo.player")
@ -12,26 +12,50 @@ Spectator.describe Invidious::Hashtag do
# Some basic verifications
expect(typeof(info)).to eq(Hash(String, JSON::Any))
expect(info["shortDescription"].as_s).to eq(
"Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch."
)
expect(info["descriptionHtml"].as_s).to eq(
"Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch."
)
expect(info["videoType"].as_s).to eq("Scheduled")
# Basic video infos
expect(info["title"].as_s).to eq("Xenoblade Chronicles 3 Nintendo Direct")
expect(info["views"].as_i).to eq(160)
expect(info["likes"].as_i).to eq(2_283)
expect(info["lengthSeconds"].as_i).to eq(0_i64)
expect(info["published"].as_s).to eq("2022-06-22T14:00:00Z") # Unix 1655906400
expect(info["genre"].as_s).to eq("Gaming")
expect(info["genreUrl"].raw).to be_nil
expect(info["genreUcid"].as_s).to be_empty
expect(info["license"].as_s).to be_empty
# Extra video infos
expect(info["authorThumbnail"].as_s).to eq(
"https://yt3.ggpht.com/ytc/AKedOLTt4vtjREUUNdHlyu9c4gtJjG90M9jQheRlLKy44A=s48-c-k-c0x00ffffff-no-rj"
expect(info["allowedRegions"].as_a).to_not be_empty
expect(info["allowedRegions"].as_a.size).to eq(249)
expect(info["allowedRegions"].as_a).to contain(
"AD", "BA", "BB", "BW", "BY", "EG", "GG", "HN", "NP", "NR", "TR",
"TT", "TV", "TW", "TZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU",
"WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"
)
expect(info["authorVerified"].as_bool).to be_true
expect(info["subCountText"].as_s).to eq("8.5M")
expect(info["keywords"].as_a).to_not be_empty
expect(info["keywords"].as_a.size).to eq(11)
expect(info["keywords"].as_a).to contain_exactly(
"nintendo",
"game",
"gameplay",
"fun",
"video game",
"action",
"adventure",
"rpg",
"play",
"switch",
"nintendo switch"
).in_any_order
expect(info["allowRatings"].as_bool).to be_true
expect(info["isFamilyFriendly"].as_bool).to be_true
expect(info["isListed"].as_bool).to be_true
expect(info["isUpcoming"].as_bool).to be_true
# Related videos
expect(info["relatedVideos"].as_a.size).to eq(20)
@ -50,6 +74,32 @@ Spectator.describe Invidious::Hashtag do
expect(info["relatedVideos"][16]["view_count"].as_s).to eq("53510")
expect(info["relatedVideos"][16]["short_view_count"].as_s).to eq("53K")
expect(info["relatedVideos"][16]["author_verified"].as_s).to eq("true")
# Description
description = "Tune in on 6/22 at 7 a.m. PT for a livestreamed Xenoblade Chronicles 3 Direct presentation featuring roughly 20 minutes of information about the upcoming RPG adventure for Nintendo Switch."
expect(info["description"].as_s).to eq(description)
expect(info["shortDescription"].as_s).to eq(description)
expect(info["descriptionHtml"].as_s).to eq(description)
# Video metadata
expect(info["genre"].as_s).to eq("Gaming")
expect(info["genreUcid"].as_s).to be_empty
expect(info["license"].as_s).to be_empty
# Author infos
expect(info["author"].as_s).to eq("Nintendo")
expect(info["ucid"].as_s).to eq("UCGIY_O-8vW4rfX98KlMkvRg")
expect(info["authorThumbnail"].as_s).to eq(
"https://yt3.ggpht.com/ytc/AKedOLTt4vtjREUUNdHlyu9c4gtJjG90M9jQheRlLKy44A=s48-c-k-c0x00ffffff-no-rj"
)
expect(info["authorVerified"].as_bool).to be_true
expect(info["subCountText"].as_s).to eq("8.5M")
end
it "parses scheduled livestreams data (test 2)" do
@ -63,34 +113,63 @@ Spectator.describe Invidious::Hashtag do
# Some basic verifications
expect(typeof(info)).to eq(Hash(String, JSON::Any))
expect(info["shortDescription"].as_s).to start_with(
<<-TXT
PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick.
expect(info["videoType"].as_s).to eq("Scheduled")
Join the channel to get exclusive access to perks: https://bit.ly/3Q9rSQL
TXT
)
expect(info["descriptionHtml"].as_s).to start_with(
<<-TXT
PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick.
Join the channel to get exclusive access to perks: <a href="https://bit.ly/3Q9rSQL">bit.ly/3Q9rSQL</a>
TXT
)
# Basic video infos
expect(info["title"].as_s).to eq("The Truth About Greenpeace w/ Dr. Patrick Moore | PBD Podcast | Ep. 171")
expect(info["views"].as_i).to eq(24)
expect(info["likes"].as_i).to eq(22)
expect(info["lengthSeconds"].as_i).to eq(0_i64)
expect(info["published"].as_s).to eq("2022-07-14T13:00:00Z") # Unix 1657803600
expect(info["genre"].as_s).to eq("Entertainment")
expect(info["genreUrl"].raw).to be_nil
expect(info["genreUcid"].as_s).to be_empty
expect(info["license"].as_s).to be_empty
# Extra video infos
expect(info["authorThumbnail"].as_s).to eq(
"https://yt3.ggpht.com/61ArDiQshJrvSXcGLhpFfIO3hlMabe2fksitcf6oGob0Mdr5gztdkXxRljICUodL4iuTSrtxW4A=s48-c-k-c0x00ffffff-no-rj"
expect(info["allowedRegions"].as_a).to_not be_empty
expect(info["allowedRegions"].as_a.size).to eq(249)
expect(info["allowedRegions"].as_a).to contain(
"AD", "AR", "BA", "BT", "CZ", "FO", "GL", "IO", "KE", "KH", "LS",
"LT", "MP", "NO", "PR", "RO", "SE", "SK", "SS", "SX", "SZ", "ZW"
)
expect(info["authorVerified"].as_bool).to be_false
expect(info["subCountText"].as_s).to eq("227K")
expect(info["keywords"].as_a).to_not be_empty
expect(info["keywords"].as_a.size).to eq(25)
expect(info["keywords"].as_a).to contain_exactly(
"Patrick Bet-David",
"Valeutainment",
"The BetDavid Podcast",
"The BetDavid Show",
"Betdavid",
"PBD",
"BetDavid show",
"Betdavid podcast",
"podcast betdavid",
"podcast patrick",
"patrick bet david podcast",
"Valuetainment podcast",
"Entrepreneurs",
"Entrepreneurship",
"Entrepreneur Motivation",
"Entrepreneur Advice",
"Startup Entrepreneurs",
"valuetainment",
"patrick bet david",
"PBD podcast",
"Betdavid show",
"Betdavid Podcast",
"Podcast Betdavid",
"Show Betdavid",
"PBDPodcast"
).in_any_order
expect(info["allowRatings"].as_bool).to be_true
expect(info["isFamilyFriendly"].as_bool).to be_true
expect(info["isListed"].as_bool).to be_true
expect(info["isUpcoming"].as_bool).to be_true
# Related videos
expect(info["relatedVideos"].as_a.size).to eq(20)
@ -109,5 +188,41 @@ Spectator.describe Invidious::Hashtag do
expect(info["relatedVideos"][9]["view_count"]).to eq("26432")
expect(info["relatedVideos"][9]["short_view_count"]).to eq("26K")
expect(info["relatedVideos"][9]["author_verified"]).to eq("true")
# Description
description_start_text = <<-TXT
PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick.
Join the channel to get exclusive access to perks: https://bit.ly/3Q9rSQL
TXT
expect(info["description"].as_s).to start_with(description_start_text)
expect(info["shortDescription"].as_s).to start_with(description_start_text)
expect(info["descriptionHtml"].as_s).to start_with(
<<-TXT
PBD Podcast Episode 171. In this episode, Patrick Bet-David is joined by Dr. Patrick Moore and Adam Sosnick.
Join the channel to get exclusive access to perks: <a href="https://bit.ly/3Q9rSQL">bit.ly/3Q9rSQL</a>
TXT
)
# Video metadata
expect(info["genre"].as_s).to eq("Entertainment")
expect(info["genreUcid"].as_s).to be_empty
expect(info["license"].as_s).to be_empty
# Author infos
expect(info["author"].as_s).to eq("PBD Podcast")
expect(info["ucid"].as_s).to eq("UCGX7nGXpz-CmO_Arg-cgJ7A")
expect(info["authorThumbnail"].as_s).to eq(
"https://yt3.ggpht.com/61ArDiQshJrvSXcGLhpFfIO3hlMabe2fksitcf6oGob0Mdr5gztdkXxRljICUodL4iuTSrtxW4A=s48-c-k-c0x00ffffff-no-rj"
)
expect(info["authorVerified"].as_bool).to be_false
expect(info["subCountText"].as_s).to eq("227K")
end
end

@ -12,6 +12,7 @@ require "../src/invidious/helpers/logger"
require "../src/invidious/helpers/utils"
require "../src/invidious/videos"
require "../src/invidious/videos/*"
require "../src/invidious/comments"
require "../src/invidious/helpers/serialized_yt_data"

@ -5,6 +5,7 @@ require "protodec/utils"
require "yaml"
require "../src/invidious/helpers/*"
require "../src/invidious/channels/*"
require "../src/invidious/videos/caption"
require "../src/invidious/videos"
require "../src/invidious/comments"
require "../src/invidious/playlists"

@ -37,6 +37,9 @@ require "./invidious/database/migrations/*"
require "./invidious/helpers/*"
require "./invidious/yt_backend/*"
require "./invidious/frontend/*"
require "./invidious/videos/*"
require "./invidious/jsonify/**"
require "./invidious/*"
require "./invidious/channels/*"

@ -29,7 +29,7 @@ struct ChannelVideo
json.field "title", self.title
json.field "videoId", self.id
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "lengthSeconds", self.length_seconds

@ -138,7 +138,7 @@ def fetch_channel_community(ucid, continuation, locale, format, thin_mode)
json.field "title", video_title
json.field "videoId", video_id
json.field "videoThumbnails" do
generate_thumbnails(json, video_id)
Invidious::JSONify::APIv1.thumbnails(json, video_id)
end
json.field "lengthSeconds", decode_length_seconds(attachment["lengthText"]["simpleText"].as_s)

@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage
getter full_videos : Array(Hash(String, JSON::Any))
getter video_streams : Array(Hash(String, JSON::Any))
getter audio_streams : Array(Hash(String, JSON::Any))
getter captions : Array(Caption)
getter captions : Array(Invidious::Videos::Caption)
def initialize(
@full_videos,
@ -50,7 +50,7 @@ module Invidious::Frontend::WatchPage
video_assets.full_videos.each do |option|
mimetype = option["mimeType"].as_s.split(";")[0]
height = itag_to_metadata?(option["itag"]).try &.["height"]?
height = Invidious::Videos::Formats.itag_to_metadata?(option["itag"]).try &.["height"]?
value = {"itag": option["itag"], "ext": mimetype.split("/")[1]}.to_json

@ -76,7 +76,7 @@ struct SearchVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
json.field "description", html_to_content(self.description_html)
@ -155,7 +155,7 @@ struct SearchPlaylist
json.field "lengthSeconds", video.length_seconds
json.field "videoThumbnails" do
generate_thumbnails(json, video.id)
Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
end
end

@ -0,0 +1,18 @@
require "json"
module Invidious::JSONify::APIv1
extend self
def thumbnails(json : JSON::Builder, id : String)
json.array do
build_thumbnails(id).each do |thumbnail|
json.object do
json.field "quality", thumbnail[:name]
json.field "url", "#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg"
json.field "width", thumbnail[:width]
json.field "height", thumbnail[:height]
end
end
end
end
end

@ -0,0 +1,251 @@
require "json"
module Invidious::JSONify::APIv1
extend self
def video(video : Video, json : JSON::Builder, *, locale : String?)
json.object do
json.field "type", video.video_type
json.field "title", video.title
json.field "videoId", video.id
json.field "error", video.info["reason"] if video.info["reason"]?
json.field "videoThumbnails" do
self.thumbnails(json, video.id)
end
json.field "storyboards" do
self.storyboards(json, video.id, video.storyboards)
end
json.field "description", video.description
json.field "descriptionHtml", video.description_html
json.field "published", video.published.to_unix
json.field "publishedText", translate(locale, "`x` ago", recode_date(video.published, locale))
json.field "keywords", video.keywords
json.field "viewCount", video.views
json.field "likeCount", video.likes
json.field "dislikeCount", 0_i64
json.field "paid", video.paid
json.field "premium", video.premium
json.field "isFamilyFriendly", video.is_family_friendly
json.field "allowedRegions", video.allowed_regions
json.field "genre", video.genre
json.field "genreUrl", video.genre_url
json.field "author", video.author
json.field "authorId", video.ucid
json.field "authorUrl", "/channel/#{video.ucid}"
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", video.author_thumbnail.gsub(/=s\d+/, "=s#{quality}")
json.field "width", quality
json.field "height", quality
end
end
end
end
json.field "subCountText", video.sub_count_text
json.field "lengthSeconds", video.length_seconds
json.field "allowRatings", video.allow_ratings
json.field "rating", 0_i64
json.field "isListed", video.is_listed
json.field "liveNow", video.live_now
json.field "isUpcoming", video.is_upcoming
if video.premiere_timestamp
json.field "premiereTimestamp", video.premiere_timestamp.try &.to_unix
end
if hlsvp = video.hls_manifest_url
hlsvp = hlsvp.gsub("https://manifest.googlevideo.com", HOST_URL)
json.field "hlsUrl", hlsvp
end
json.field "dashUrl", "#{HOST_URL}/api/manifest/dash/id/#{video.id}"
json.field "adaptiveFormats" do
json.array do
video.adaptive_fmts.each do |fmt|
json.object do
# Only available on regular videos, not livestreams/OTF streams
if init_range = fmt["initRange"]?
json.field "init", "#{init_range["start"]}-#{init_range["end"]}"
end
if index_range = fmt["indexRange"]?
json.field "index", "#{index_range["start"]}-#{index_range["end"]}"
end
# Not available on MPEG-4 Timed Text (`text/mp4`) streams (livestreams only)
json.field "bitrate", fmt["bitrate"].as_i.to_s if fmt["bitrate"]?
json.field "url", fmt["url"]
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "clen", fmt["contentLength"]? || "-1"
# Last modified is a unix timestamp with µS, with the dot omitted.
# E.g: 1638056732(.)141582
#
# On livestreams, it's not present, so always fall back to the
# current unix timestamp (up to mS precision) for compatibility.
last_modified = fmt["lastModified"]?
last_modified ||= "#{Time.utc.to_unix_ms.to_s}000"
json.field "lmt", last_modified
json.field "projectionType", fmt["projectionType"]
if fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
# Livestream chunk infos
json.field "targetDurationSec", fmt["targetDurationSec"].as_i if fmt.has_key?("targetDurationSec")
json.field "maxDvrDurationSec", fmt["maxDvrDurationSec"].as_i if fmt.has_key?("maxDvrDurationSec")
# Audio-related data
json.field "audioQuality", fmt["audioQuality"] if fmt.has_key?("audioQuality")
json.field "audioSampleRate", fmt["audioSampleRate"].as_s.to_i if fmt.has_key?("audioSampleRate")
json.field "audioChannels", fmt["audioChannels"] if fmt.has_key?("audioChannels")
# Extra misc stuff
json.field "colorInfo", fmt["colorInfo"] if fmt.has_key?("colorInfo")
json.field "captionTrack", fmt["captionTrack"] if fmt.has_key?("captionTrack")
end
end
end
end
json.field "formatStreams" do
json.array do
video.fmt_stream.each do |fmt|
json.object do
json.field "url", fmt["url"]
json.field "itag", fmt["itag"].as_i.to_s
json.field "type", fmt["mimeType"]
json.field "quality", fmt["quality"]
fmt_info = Invidious::Videos::Formats.itag_to_metadata?(fmt["itag"])
if fmt_info
fps = fmt_info["fps"]?.try &.to_i || fmt["fps"]?.try &.as_i || 30
json.field "fps", fps
json.field "container", fmt_info["ext"]
json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"]
if fmt_info["height"]?
json.field "resolution", "#{fmt_info["height"]}p"
quality_label = "#{fmt_info["height"]}p"
if fps > 30
quality_label += "60"
end
json.field "qualityLabel", quality_label
if fmt_info["width"]?
json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}"
end
end
end
end
end
end
end
json.field "captions" do
json.array do
video.captions.each do |caption|
json.object do
json.field "label", caption.name
json.field "language_code", caption.language_code
json.field "url", "/api/v1/captions/#{video.id}?label=#{URI.encode_www_form(caption.name)}"
end
end
end
end
json.field "recommendedVideos" do
json.array do
video.related_videos.each do |rv|
if rv["id"]?
json.object do
json.field "videoId", rv["id"]
json.field "title", rv["title"]
json.field "videoThumbnails" do
self.thumbnails(json, rv["id"])
end
json.field "author", rv["author"]
json.field "authorUrl", "/channel/#{rv["ucid"]?}"
json.field "authorId", rv["ucid"]?
if rv["author_thumbnail"]?
json.field "authorThumbnails" do
json.array do
qualities = {32, 48, 76, 100, 176, 512}
qualities.each do |quality|
json.object do
json.field "url", rv["author_thumbnail"].gsub(/s\d+-/, "s#{quality}-")
json.field "width", quality
json.field "height", quality
end
end
end
end
end
json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i
json.field "viewCountText", rv["short_view_count"]?
json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64
end
end
end
end
end
end
end
def storyboards(json, id, storyboards)
json.array do
storyboards.each do |storyboard|
json.object do
json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}"
json.field "templateUrl", storyboard[:url]
json.field "width", storyboard[:width]
json.field "height", storyboard[:height]
json.field "count", storyboard[:count]
json.field "interval", storyboard[:interval]
json.field "storyboardWidth", storyboard[:storyboard_width]
json.field "storyboardHeight", storyboard[:storyboard_height]
json.field "storyboardCount", storyboard[:storyboard_count]
end
end
end
end
end

@ -56,7 +56,7 @@ struct PlaylistVideo
json.field "authorUrl", "/channel/#{self.ucid}"
json.field "videoThumbnails" do
generate_thumbnails(json, self.id)
Invidious::JSONify::APIv1.thumbnails(json, self.id)
end
if index

@ -14,8 +14,6 @@ module Invidious::Routes::API::Manifest
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
haltf env, status_code: 404
rescue ex

@ -124,7 +124,7 @@ module Invidious::Routes::API::V1::Misc
json.field "videoThumbnails" do
json.array do
generate_thumbnails(json, video.id)
Invidious::JSONify::APIv1.thumbnails(json, video.id)
end
end

@ -9,9 +9,6 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
return error_json(404, ex)
rescue ex
@ -41,9 +38,6 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
@ -168,9 +162,6 @@ module Invidious::Routes::API::V1::Videos
begin
video = get_video(id, region: region)
rescue ex : VideoRedirect
env.response.headers["Location"] = env.request.resource.gsub(id, ex.video_id)
return error_json(302, "Video is unavailable", {"videoId" => ex.video_id})
rescue ex : NotFoundException
haltf env, 404
rescue ex
@ -185,7 +176,7 @@ module Invidious::Routes::API::V1::Videos
response = JSON.build do |json|
json.object do
json.field "storyboards" do
generate_storyboards(json, id, storyboards)
Invidious::JSONify::APIv1.storyboards(json, id, storyboards)
end
end
end

@ -131,8 +131,6 @@ module Invidious::Routes::Embed
begin
video = get_video(id, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
return error_template(404, ex)
rescue ex

@ -61,8 +61,6 @@ module Invidious::Routes::Watch
begin
video = get_video(id, region: params.region)
rescue ex : VideoRedirect
return env.redirect env.request.resource.gsub(id, ex.video_id)
rescue ex : NotFoundException
LOGGER.error("get_video not found: #{id} : #{ex.message}")
return error_template(404, ex)

File diff suppressed because it is too large Load Diff

@ -0,0 +1,168 @@
require "json"
module Invidious::Videos
struct Caption
property name : String
property language_code : String
property base_url : String
def initialize(@name, @language_code, @base_url)
end
# Parse the JSON structure from Youtube
def self.from_yt_json(container : JSON::Any) : Array(Caption)
caption_tracks = container
.dig?("playerCaptionsTracklistRenderer", "captionTracks")
.try &.as_a
captions_list = [] of Caption
return captions_list if caption_tracks.nil?
caption_tracks.each do |caption|
name = caption["name"]["simpleText"]? || caption["name"]["runs"][0]["text"]
name = name.to_s.split(" - ")[0]
language_code = caption["languageCode"].to_s
base_url = caption["baseUrl"].to_s
captions_list << Caption.new(name, language_code, base_url)
end
return captions_list
end
# List of all caption languages available on Youtube.
LANGUAGES = {
"",
"English",
"English (auto-generated)",
"English (United Kingdom)",
"English (United States)",
"Afrikaans",
"Albanian",
"Amharic",
"Arabic",
"Armenian",
"Azerbaijani",
"Bangla",
"Basque",
"Belarusian",
"Bosnian",
"Bulgarian",
"Burmese",
"Cantonese (Hong Kong)",
"Catalan",
"Cebuano",
"Chinese",
"Chinese (China)",
"Chinese (Hong Kong)",
"Chinese (Simplified)",
"Chinese (Taiwan)",
"Chinese (Traditional)",
"Corsican",
"Croatian",
"Czech",
"Danish",
"Dutch",
"Dutch (auto-generated)",
"Esperanto",
"Estonian",
"Filipino",
"Finnish",
"French",
"French (auto-generated)",
"Galician",
"Georgian",
"German",
"German (auto-generated)",
"Greek",
"Gujarati",
"Haitian Creole",
"Hausa",
"Hawaiian",
"Hebrew",
"Hindi",
"Hmong",
"Hungarian",
"Icelandic",
"Igbo",
"Indonesian",
"Indonesian (auto-generated)",
"Interlingue",
"Irish",
"Italian",
"Italian (auto-generated)",
"Japanese",
"Japanese (auto-generated)",
"Javanese",
"Kannada",
"Kazakh",
"Khmer",
"Korean",
"Korean (auto-generated)",
"Kurdish",
"Kyrgyz",
"Lao",
"Latin",
"Latvian",
"Lithuanian",
"Luxembourgish",
"Macedonian",
"Malagasy",
"Malay",
"Malayalam",
"Maltese",
"Maori",
"Marathi",
"Mongolian",
"Nepali",
"Norwegian Bokmål",
"Nyanja",
"Pashto",
"Persian",
"Polish",
"Portuguese",
"Portuguese (auto-generated)",
"Portuguese (Brazil)",
"Punjabi",
"Romanian",
"Russian",
"Russian (auto-generated)",
"Samoan",
"Scottish Gaelic",
"Serbian",
"Shona",
"Sindhi",
"Sinhala",
"Slovak",
"Slovenian",
"Somali",
"Southern Sotho",
"Spanish",
"Spanish (auto-generated)",
"Spanish (Latin America)",
"Spanish (Mexico)",
"Spanish (Spain)",
"Sundanese",
"Swahili",
"Swedish",
"Tajik",
"Tamil",
"Telugu",
"Thai",
"Turkish",
"Turkish (auto-generated)",
"Ukrainian",
"Urdu",
"Uzbek",
"Vietnamese",
"Vietnamese (auto-generated)",
"Welsh",
"Western Frisian",
"Xhosa",
"Yiddish",
"Yoruba",
"Zulu",
}
end
end

@ -0,0 +1,116 @@
module Invidious::Videos::Formats
def self.itag_to_metadata?(itag : JSON::Any)
return FORMATS[itag.to_s]?
end
# See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476
private FORMATS = {
"5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
"6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"},
"13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"},
"17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"},
"18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"},
"22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"},
"37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
# 3D videos
"82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"},
"100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"},
"101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
"102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"},
# Apple HTTP Live Streaming
"91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"},
"95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
"96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"},
"132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"},
"151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"},
# DASH mp4 video
"133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"},
"134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"},
"135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
"136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"},
"137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"},
"138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
"160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"},
"212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"},
"264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"},
"298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
"299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60},
"266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"},
# Dash mp4 audio
"139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"},
"140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"},
"141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"},
"256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
"258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"},
"325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"},
"328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"},
# Dash webm
"167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"},
"278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"},
"242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"},
"243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"},
"244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"},
"247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"},
"248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"},
"271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"},
# itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
"272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
"302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"},
"315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
"337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60},
# Dash webm audio
"171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128},
"172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256},
# Dash webm audio with opus inside
"249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50},
"250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70},
"251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160},
# av01 video only formats sometimes served with "unknown" codecs
"394" => {"ext" => "mp4", "height" => 144, "vcodec" => "av01.0.05M.08"},
"395" => {"ext" => "mp4", "height" => 240, "vcodec" => "av01.0.05M.08"},
"396" => {"ext" => "mp4", "height" => 360, "vcodec" => "av01.0.05M.08"},
"397" => {"ext" => "mp4", "height" => 480, "vcodec" => "av01.0.05M.08"},
}
end

@ -0,0 +1,369 @@
require "json"
# Use to parse both "compactVideoRenderer" and "endScreenVideoRenderer".
# The former is preferred as it has more videos in it. The second has
# the same 11 first entries as the compact rendered.
#
# TODO: "compactRadioRenderer" (Mix) and
# TODO: Use a proper struct/class instead of a hacky JSON object
def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)?
return nil if !related["videoId"]?
# The compact renderer has video length in seconds, where the end
# screen rendered has a full text version ("42:40")
length = related["lengthInSeconds"]?.try &.as_i.to_s
length ||= related.dig?("lengthText", "simpleText").try do |box|
decode_length_seconds(box.as_s).to_s
end
# Both have "short", so the "long" option shouldn't be required
channel_info = (related["shortBylineText"]? || related["longBylineText"]?)
.try &.dig?("runs", 0)
author = channel_info.try &.dig?("text")
author_verified = has_verified_badge?(related["ownerBadges"]?).to_s
ucid = channel_info.try { |ci| HelperExtractors.get_browse_id(ci) }
# "4,088,033 views", only available on compact renderer
# and when video is not a livestream
view_count = related.dig?("viewCountText", "simpleText")
.try &.as_s.gsub(/\D/, "")
short_view_count = related.try do |r|
HelperExtractors.get_short_view_count(r).to_s
end
LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container")
# TODO: when refactoring video types, make a struct for related videos
# or reuse an existing type, if that fits.
return {
"id" => related["videoId"],
"title" => related["title"]["simpleText"],
"author" => author || JSON::Any.new(""),
"ucid" => JSON::Any.new(ucid || ""),
"length_seconds" => JSON::Any.new(length || "0"),
"view_count" => JSON::Any.new(view_count || "0"),
"short_view_count" => JSON::Any.new(short_view_count || "0"),
"author_verified" => JSON::Any.new(author_verified),
}
end
def extract_video_info(video_id : String, proxy_region : String? = nil)
# Init client config for the API
client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region)
# Fetch data from the player endpoint
player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config)
playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s
if playability_status != "OK"
subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason")
reason = subreason.try &.[]?("simpleText").try &.as_s
reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("")
reason ||= player_response.dig("playabilityStatus", "reason").as_s
# Stop here if video is not a scheduled livestream
if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
return {
"version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64),
"reason" => JSON::Any.new(reason),
}
end
elsif video_id != player_response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
raise VideoNotAvailableException.new("The video returned by YouTube isn't the requested one. (WEB client)")
else
reason = nil
end
# Don't fetch the next endpoint if the video is unavailable.
if {"OK", "LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status)
next_response = YoutubeAPI.next({"videoId": video_id, "params": ""})
player_response = player_response.merge(next_response)
end
params = parse_video_info(video_id, player_response)
params["reason"] = JSON::Any.new(reason) if reason
new_player_response = nil
if reason.nil?
# Fetch the video streams using an Android client in order to get the
# decrypted URLs and maybe fix throttling issues (#2194). See the
# following issue for an explanation about decrypted URLs:
# https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
client_config.client_type = YoutubeAPI::ClientType::Android
new_player_response = try_fetch_streaming_data(video_id, client_config)
elsif !reason.includes?("your country") # Handled separately
# The Android embedded client could help here
client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed
new_player_response = try_fetch_streaming_data(video_id, client_config)
end
# Last hope
if new_player_response.nil?
client_config.client_type = YoutubeAPI::ClientType::TvHtml5ScreenEmbed
new_player_response = try_fetch_streaming_data(video_id, client_config)
end
# Replace player response and reset reason
if !new_player_response.nil?
player_response = new_player_response
params.delete("reason")
end
{"captions", "playabilityStatus", "playerConfig", "storyboards", "streamingData"}.each do |f|
params[f] = player_response[f] if player_response[f]?
end
# Data structure version, for cache control
params["version"] = JSON::Any.new(Video::SCHEMA_VERSION.to_i64)
return params
end
def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConfig) : Hash(String, JSON::Any)?
LOGGER.debug("try_fetch_streaming_data: [#{id}] Using #{client_config.client_type} client.")
response = YoutubeAPI.player(video_id: id, params: "", client_config: client_config)
playability_status = response["playabilityStatus"]["status"]
LOGGER.debug("try_fetch_streaming_data: [#{id}] Got playabilityStatus == #{playability_status}.")
if id != response.dig("videoDetails", "videoId")
# YouTube may return a different video player response than expected.
# See: https://github.com/TeamNewPipe/NewPipe/issues/8713
raise VideoNotAvailableException.new(
"The video returned by YouTube isn't the requested one. (#{client_config.client_type} client)"
)
elsif playability_status == "OK"
return response
else
return nil
end
end
def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any)
# Top level elements
main_results = player_response.dig?("contents", "twoColumnWatchNextResults")
raise BrokenTubeException.new("twoColumnWatchNextResults") if !main_results
# Primary results are not available on Music videos
# See: https://github.com/iv-org/invidious/pull/3238#issuecomment-1207193725
if primary_results = main_results.dig?("results", "results", "contents")
video_primary_renderer = primary_results
.as_a.find(&.["videoPrimaryInfoRenderer"]?)
.try &.["videoPrimaryInfoRenderer"]
video_secondary_renderer = primary_results
.as_a.find(&.["videoSecondaryInfoRenderer"]?)
.try &.["videoSecondaryInfoRenderer"]
raise BrokenTubeException.new("videoPrimaryInfoRenderer") if !video_primary_renderer
raise BrokenTubeException.new("videoSecondaryInfoRenderer") if !video_secondary_renderer
end
video_details = player_response.dig?("videoDetails")
microformat = player_response.dig?("microformat", "playerMicroformatRenderer")
raise BrokenTubeException.new("videoDetails") if !video_details
raise BrokenTubeException.new("microformat") if !microformat
# Basic video infos
title = video_details["title"]?.try &.as_s
# We have to try to extract viewCount from videoPrimaryInfoRenderer first,
# then from videoDetails, as the latter is "0" for livestreams (we want
# to get the amount of viewers watching).
views_txt = video_primary_renderer
.try &.dig?("viewCount", "videoViewCountRenderer", "viewCount", "runs", 0, "text")
views_txt ||= video_details["viewCount"]?
views = views_txt.try &.as_s.gsub(/\D/, "").to_i64?
length_txt = (microformat["lengthSeconds"]? || video_details["lengthSeconds"])
.try &.as_s.to_i64
published = microformat["publishDate"]?
.try { |t| Time.parse(t.as_s, "%Y-%m-%d", Time::Location::UTC) } || Time.utc
premiere_timestamp = microformat.dig?("liveBroadcastDetails", "startTimestamp")
.try { |t| Time.parse_rfc3339(t.as_s) }
live_now = microformat.dig?("liveBroadcastDetails", "isLiveNow")
.try &.as_bool || false
# Extra video infos
allowed_regions = microformat["availableCountries"]?
.try &.as_a.map &.as_s || [] of String
allow_ratings = video_details["allowRatings"]?.try &.as_bool
family_friendly = microformat["isFamilySafe"].try &.as_bool
is_listed = video_details["isCrawlable"]?.try &.as_bool
is_upcoming = video_details["isUpcoming"]?.try &.as_bool
keywords = video_details["keywords"]?
.try &.as_a.map &.as_s || [] of String
# Related videos
LOGGER.debug("extract_video_info: parsing related videos...")
related = [] of JSON::Any
# Parse "compactVideoRenderer" items (under secondary results)
secondary_results = main_results
.dig?("secondaryResults", "secondaryResults", "results")
secondary_results.try &.as_a.each do |element|
if item = element["compactVideoRenderer"]?
related_video = parse_related_video(item)
related << JSON::Any.new(related_video) if related_video
end
end
# If nothing was found previously, fall back to end screen renderer
if related.empty?
# Container for "endScreenVideoRenderer" items
player_overlays = player_response.dig?(
"playerOverlays", "playerOverlayRenderer",
"endScreen", "watchNextEndScreenRenderer", "results"
)
player_overlays.try &.as_a.each do |element|
if item = element["endScreenVideoRenderer"]?
related_video = parse_related_video(item)
related << JSON::Any.new(related_video) if related_video
end
end
end
# Likes
toplevel_buttons = video_primary_renderer
.try &.dig?("videoActions", "menuRenderer", "topLevelButtons")
if toplevel_buttons
likes_button = toplevel_buttons.try &.as_a
.find(&.dig?("toggleButtonRenderer", "defaultIcon", "iconType").=== "LIKE")
.try &.["toggleButtonRenderer"]
# New format as of september 2022
likes_button ||= toplevel_buttons.try &.as_a
.find(&.["segmentedLikeDislikeButtonRenderer"]?)
.try &.dig?(
"segmentedLikeDislikeButtonRenderer",
"likeButton", "toggleButtonRenderer"
)
if likes_button
# Note: The like count from `toggledText` is off by one, as it would
# represent the new like count in the event where the user clicks on "like".
likes_txt = (likes_button["defaultText"]? || likes_button["toggledText"]?)
.try &.dig?("accessibility", "accessibilityData", "label")
likes = likes_txt.as_s.gsub(/\D/, "").to_i64? if likes_txt
LOGGER.trace("extract_video_info: Found \"likes\" button. Button text is \"#{likes_txt}\"")
LOGGER.debug("extract_video_info: Likes count is #{likes}") if likes
end
end
# Description
description = microformat.dig?("description", "simpleText").try &.as_s || ""
short_description = player_response.dig?("videoDetails", "shortDescription")
description_html = video_secondary_renderer.try &.dig?("description", "runs")
.try &.as_a.try { |t| content_to_comment_html(t, video_id) }
# Video metadata
metadata = video_secondary_renderer
.try &.dig?("metadataRowContainer", "metadataRowContainerRenderer", "rows")
.try &.as_a
genre = microformat["category"]?
genre_ucid = nil
license = nil
metadata.try &.each do |row|
metadata_title = extract_text(row.dig?("metadataRowRenderer", "title"))
contents = row.dig?("metadataRowRenderer", "contents", 0)
if metadata_title == "Category"
contents = contents.try &.dig?("runs", 0)
genre = contents.try &.["text"]?
genre_ucid = contents.try &.dig?("navigationEndpoint", "browseEndpoint", "browseId")
elsif metadata_title == "License"
license = contents.try &.dig?("runs", 0, "text")
elsif metadata_title == "Licensed to YouTube by"
license = contents.try &.["simpleText"]?
end
end
# Author infos
author = video_details["author"]?.try &.as_s
ucid = video_details["channelId"]?.try &.as_s
if author_info = video_secondary_renderer.try &.dig?("owner", "videoOwnerRenderer")
author_thumbnail = author_info.dig?("thumbnail", "thumbnails", 0, "url")
author_verified = has_verified_badge?(author_info["badges"]?)
subs_text = author_info["subscriberCountText"]?
.try { |t| t["simpleText"]? || t.dig?("runs", 0, "text") }
.try &.as_s.split(" ", 2)[0]
end
# Return data
if live_now
video_type = VideoType::Livestream
elsif !premiere_timestamp.nil?
video_type = VideoType::Scheduled
published = premiere_timestamp || Time.utc
else
video_type = VideoType::Video
end
params = {
"videoType" => JSON::Any.new(video_type.to_s),
# Basic video infos
"title" => JSON::Any.new(title || ""),
"views" => JSON::Any.new(views || 0_i64),
"likes" => JSON::Any.new(likes || 0_i64),
"lengthSeconds" => JSON::Any.new(length_txt || 0_i64),
"published" => JSON::Any.new(published.to_rfc3339),
# Extra video infos
"allowedRegions" => JSON::Any.new(allowed_regions.map { |v| JSON::Any.new(v) }),
"allowRatings" => JSON::Any.new(allow_ratings || false),
"isFamilyFriendly" => JSON::Any.new(family_friendly || false),
"isListed" => JSON::Any.new(is_listed || false),
"isUpcoming" => JSON::Any.new(is_upcoming || false),
"keywords" => JSON::Any.new(keywords.map { |v| JSON::Any.new(v) }),
# Related videos
"relatedVideos" => JSON::Any.new(related),
# Description
"description" => JSON::Any.new(description || ""),
"descriptionHtml" => JSON::Any.new(description_html || "<p></p>"),
"shortDescription" => JSON::Any.new(short_description.try &.as_s || nil),
# Video metadata
"genre" => JSON::Any.new(genre.try &.as_s || ""),
"genreUcid" => JSON::Any.new(genre_ucid.try &.as_s || ""),
"license" => JSON::Any.new(license.try &.as_s || ""),
# Author infos
"author" => JSON::Any.new(author || ""),
"ucid" => JSON::Any.new(ucid || ""),
"authorThumbnail" => JSON::Any.new(author_thumbnail.try &.as_s || ""),
"authorVerified" => JSON::Any.new(author_verified || false),
"subCountText" => JSON::Any.new(subs_text || "-"),
}
return params
end

@ -0,0 +1,27 @@
# List of geographical regions that Youtube recognizes.
# This is used to determine if a video is either restricted to a list
# of allowed regions (= whitelisted) or if it can't be watched in
# a set of regions (= blacklisted).
REGIONS = {
"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT",
"AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI",
"BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY",
"BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN",
"CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM",
"DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK",
"FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL",
"GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM",
"HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR",
"IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN",
"KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS",
"LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK",
"ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW",
"MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP",
"NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM",
"PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW",
"SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM",
"SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF",
"TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW",
"TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI",
"VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW",
}

@ -0,0 +1,156 @@
struct VideoPreferences
include JSON::Serializable
property annotations : Bool
property autoplay : Bool
property comments : Array(String)
property continue : Bool
property continue_autoplay : Bool
property controls : Bool
property listen : Bool
property local : Bool
property preferred_captions : Array(String)
property player_style : String
property quality : String
property quality_dash : String
property raw : Bool
property region : String?
property related_videos : Bool
property speed : Float32 | Float64
property video_end : Float64 | Int32
property video_loop : Bool
property extend_desc : Bool
property video_start : Float64 | Int32
property volume : Int32
property vr_mode : Bool
property save_player_pos : Bool
end
def process_video_params(query, preferences)
annotations = query["iv_load_policy"]?.try &.to_i?
autoplay = query["autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
comments = query["comments"]?.try &.split(",").map(&.downcase)
continue = query["continue"]?.try { |q| (q == "true" || q == "1").to_unsafe }
continue_autoplay = query["continue_autoplay"]?.try { |q| (q == "true" || q == "1").to_unsafe }
listen = query["listen"]?.try { |q| (q == "true" || q == "1").to_unsafe }
local = query["local"]?.try { |q| (q == "true" || q == "1").to_unsafe }
player_style = query["player_style"]?
preferred_captions = query["subtitles"]?.try &.split(",").map(&.downcase)
quality = query["quality"]?
quality_dash = query["quality_dash"]?
region = query["region"]?
related_videos = query["related_videos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
speed = query["speed"]?.try &.rchop("x").to_f?
video_loop = query["loop"]?.try { |q| (q == "true" || q == "1").to_unsafe }
extend_desc = query["extend_desc"]?.try { |q| (q == "true" || q == "1").to_unsafe }
volume = query["volume"]?.try &.to_i?
vr_mode = query["vr_mode"]?.try { |q| (q == "true" || q == "1").to_unsafe }
save_player_pos = query["save_player_pos"]?.try { |q| (q == "true" || q == "1").to_unsafe }
if preferences
# region ||= preferences.region
annotations ||= preferences.annotations.to_unsafe
autoplay ||= preferences.autoplay.to_unsafe
comments ||= preferences.comments
continue ||= preferences.continue.to_unsafe
continue_autoplay ||= preferences.continue_autoplay.to_unsafe
listen ||= preferences.listen.to_unsafe
local ||= preferences.local.to_unsafe
player_style ||= preferences.player_style
preferred_captions ||= preferences.captions
quality ||= preferences.quality
quality_dash ||= preferences.quality_dash
related_videos ||= preferences.related_videos.to_unsafe
speed ||= preferences.speed
video_loop ||= preferences.video_loop.to_unsafe
extend_desc ||= preferences.extend_desc.to_unsafe
volume ||= preferences.volume
vr_mode ||= preferences.vr_mode.to_unsafe
save_player_pos ||= preferences.save_player_pos.to_unsafe
end
annotations ||= CONFIG.default_user_preferences.annotations.to_unsafe
autoplay ||= CONFIG.default_user_preferences.autoplay.to_unsafe
comments ||= CONFIG.default_user_preferences.comments
continue ||= CONFIG.default_user_preferences.continue.to_unsafe
continue_autoplay ||= CONFIG.default_user_preferences.continue_autoplay.to_unsafe
listen ||= CONFIG.default_user_preferences.listen.to_unsafe
local ||= CONFIG.default_user_preferences.local.to_unsafe
player_style ||= CONFIG.default_user_preferences.player_style
preferred_captions ||= CONFIG.default_user_preferences.captions
quality ||= CONFIG.default_user_preferences.quality
quality_dash ||= CONFIG.default_user_preferences.quality_dash
related_videos ||= CONFIG.default_user_preferences.related_videos.to_unsafe
speed ||= CONFIG.default_user_preferences.speed
video_loop ||= CONFIG.default_user_preferences.video_loop.to_unsafe
extend_desc ||= CONFIG.default_user_preferences.extend_desc.to_unsafe
volume ||= CONFIG.default_user_preferences.volume
vr_mode ||= CONFIG.default_user_preferences.vr_mode.to_unsafe
save_player_pos ||= CONFIG.default_user_preferences.save_player_pos.to_unsafe
annotations = annotations == 1
autoplay = autoplay == 1
continue = continue == 1
continue_autoplay = continue_autoplay == 1
listen = listen == 1
local = local == 1
related_videos = related_videos == 1
video_loop = video_loop == 1
extend_desc = extend_desc == 1
vr_mode = vr_mode == 1
save_player_pos = save_player_pos == 1
if CONFIG.disabled?("dash") && quality == "dash"
quality = "high"
end
if CONFIG.disabled?("local") && local
local = false
end
if start = query["t"]? || query["time_continue"]? || query["start"]?
video_start = decode_time(start)
end
video_start ||= 0
if query["end"]?
video_end = decode_time(query["end"])
end
video_end ||= -1
raw = query["raw"]?.try &.to_i?
raw ||= 0
raw = raw == 1
controls = query["controls"]?.try &.to_i?
controls ||= 1
controls = controls >= 1
params = VideoPreferences.new({
annotations: annotations,
autoplay: autoplay,
comments: comments,
continue: continue,
continue_autoplay: continue_autoplay,
controls: controls,
listen: listen,
local: local,
player_style: player_style,
preferred_captions: preferred_captions,
quality: quality,
quality_dash: quality_dash,
raw: raw,
region: region,
related_videos: related_videos,
speed: speed,
video_end: video_end,
video_loop: video_loop,
extend_desc: extend_desc,
video_start: video_start,
volume: volume,
vr_mode: vr_mode,
save_player_pos: save_player_pos,
})
return params
end

@ -89,7 +89,7 @@
<label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label>
<% preferences.captions.each_with_index do |caption, index| %>
<select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]">
<% CAPTION_LANGUAGES.each do |option| %>
<% Invidious::Videos::Caption::LANGUAGES.each do |option| %>
<option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option>
<% end %>
</select>

Loading…
Cancel
Save