@ -1,219 +1,100 @@
require " ./macros " require " ./macros "
struct Nonce struct Nonce
db_mapping ( {
include DB :: Serializable
nonce : String ,
expire : Time ,
} )
end
struct SessionId
db_mapping ( {
id : String ,
email : String ,
issued : String ,
} )
end
struct Annotation
db_mapping ( {
id : String ,
annotations : String ,
} )
end
struct ConfigPreferences
module StringToArray
def self . to_json ( value : Array ( String ) , json : JSON :: Builder )
json . array do
value . each do | element |
json . string element
end
end
end
def self . from_json ( value : JSON :: PullParser ) : Array ( String )
begin
result = [ ] of String
value . read_array do
result << HTML . escape ( value . read_string [ 0 , 100 ] )
end
rescue ex
result = [ HTML . escape ( value . read_string [ 0 , 100 ] ) , " " ]
end
result
end
def self . to_yaml ( value : Array ( String ) , yaml : YAML :: Nodes :: Builder )
property nonce : String
yaml . sequence do
property expire : Time
value . each do | element |
yaml . scalar element
end
end
end
def self . from_yaml ( ctx : YAML :: ParseContext , node : YAML :: Nodes :: Node ) : Array ( String )
begin
unless node . is_a? ( YAML :: Nodes :: Sequence )
node . raise " Expected sequence, not #{ node . class } "
end
result = [ ] of String
node . nodes . each do | item |
unless item . is_a? ( YAML :: Nodes :: Scalar )
node . raise " Expected scalar, not #{ item . class } "
end
result << HTML . escape ( item . value [ 0 , 100 ] )
end
rescue ex
if node . is_a? ( YAML :: Nodes :: Scalar )
result = [ HTML . escape ( node . value [ 0 , 100 ] ) , " " ]
else
result = [ " " , " " ]
end
end end
result
struct SessionId
end
include DB :: Serializable
end
module BoolTo
property id : String
def self . to_json ( value : String , json : JSON :: Builder )
property email : String
json . string value
property issued : String
end end
def self . from_json ( value : JSON :: PullParser ) : String
struct Annotation
begin
include DB :: Serializable
result = value . read_string
if result . empty?
CONFIG . default_user_preferences . dark_mode
else
result
end
rescue ex
if value . read_bool
" dark "
else
" light "
end
end
end
def self . to_yaml ( value : String , yaml : YAML :: Nodes :: Builder )
property id : String
yaml . scalar value
property annotations : String
end end
def self . from_yaml ( ctx : YAML :: ParseContext , node : YAML :: Nodes :: Node ) : String
struct ConfigPreferences
unless node . is_a? ( YAML :: Nodes :: Scalar )
include YAML :: Serializable
node . raise " Expected scalar, not #{ node . class } "
property annotations : Bool = false
property annotations_subscribed : Bool = false
property autoplay : Bool = false
property captions : Array ( String ) = [ " " , " " , " " ]
property comments : Array ( String ) = [ " youtube " , " " ]
property continue : Bool = false
property continue_autoplay : Bool = true
property dark_mode : String = " "
property latest_only : Bool = false
property listen : Bool = false
property local : Bool = false
property locale : String = " en-US "
property max_results : Int32 = 40
property notifications_only : Bool = false
property player_style : String = " invidious "
property quality : String = " hd720 "
property default_home : String = " Popular "
property feed_menu : Array ( String ) = [ " Popular " , " Trending " , " Subscriptions " , " Playlists " ]
property related_videos : Bool = true
property sort : String = " published "
property speed : Float32 = 1.0_f32
property thin_mode : Bool = false
property unseen_only : Bool = false
property video_loop : Bool = false
property volume : Int32 = 100
def to_tuple
{% begin %}
{
{{ * @type . instance_vars . map { | var | " #{ var . name } : #{ var . name } " . id }} }
}
{% end %}
end
end
case node . value
when " true "
" dark "
when " false "
" light "
when " "
CONFIG . default_user_preferences . dark_mode
else
node . value
end
end
end
yaml_mapping ( {
annotations : { type : Bool , default : false } ,
annotations_subscribed : { type : Bool , default : false } ,
autoplay : { type : Bool , default : false } ,
captions : { type : Array ( String ) , default : [ " " , " " , " " ] , converter : StringToArray } ,
comments : { type : Array ( String ) , default : [ " youtube " , " " ] , converter : StringToArray } ,
continue : { type : Bool , default : false } ,
continue_autoplay : { type : Bool , default : true } ,
dark_mode : { type : String , default : " " , converter : BoolToString } ,
latest_only : { type : Bool , default : false } ,
listen : { type : Bool , default : false } ,
local : { type : Bool , default : false } ,
locale : { type : String , default : " en-US " } ,
max_results : { type : Int32 , default : 40 } ,
notifications_only : { type : Bool , default : false } ,
player_style : { type : String , default : " invidious " } ,
quality : { type : String , default : " hd720 " } ,
default_home : { type : String , default : " Popular " } ,
feed_menu : { type : Array ( String ) , default : [ " Popular " , " Trending " , " Subscriptions " , " Playlists " ] } ,
related_videos : { type : Bool , default : true } ,
sort : { type : String , default : " published " } ,
speed : { type : Float32 , default : 1.0_f32 } ,
thin_mode : { type : Bool , default : false } ,
unseen_only : { type : Bool , default : false } ,
video_loop : { type : Bool , default : false } ,
volume : { type : Int32 , default : 100 } ,
} )
end end
struct Config struct Config
module ConfigPreferencesConverter
include YAML :: Serializable
def self . to_yaml ( value : Preferences , yaml : YAML :: Nodes :: Builder )
value . to_yaml ( yaml )
property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
end
property feed_threads : Int32 # Number of threads to use for updating feeds
property db : DBConfig # Database configuration
def self . from_yaml ( ctx : YAML :: ParseContext , node : YAML :: Nodes :: Node ) : Preferences
property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
Preferences . new ( * ConfigPreferences . new ( ctx , node ) . to_tuple )
property https_only : Bool ? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
end
property hmac_key : String ? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
end
property domain : String ? # Domain to be used for links to resources on the site where an absolute URL is required
property use_pubsub_feeds : Bool | Int32 = false # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
module FamilyConverter
property captcha_enabled : Bool = true
def self . to_yaml ( value : Socket :: Family , yaml : YAML :: Nodes :: Builder )
property login_enabled : Bool = true
case value
property registration_enabled : Bool = true
when Socket :: Family :: UNSPEC
property statistics_enabled : Bool = false
yaml . scalar nil
property admins : Array ( String ) = [ ] of String
when Socket :: Family :: INET
property external_port : Int32 ? = nil
yaml . scalar " ipv4 "
property default_user_preferences : ConfigPreferences
when Socket :: Family :: INET6
property dmca_content : Array ( String ) = [ ] of String # For compliance with DMCA, disables download widget using list of video IDs
yaml . scalar " ipv6 "
property check_tables : Bool = false # Check table integrity, automatically try to add any missing columns, create tables, etc.
when Socket :: Family :: UNIX
property cache_annotations : Bool = false # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
raise " Invalid socket family #{ value } "
property banner : String ? = nil # Optional banner to be displayed along top of page for announcements, etc.
end
property hsts : Bool ? = true # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
end
property disable_proxy : Bool ? | Array ( String ) ? = false # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
def self . from_yaml ( ctx : YAML :: ParseContext , node : YAML :: Nodes :: Node ) : Socket :: Family
@[ YAML :: Field ( converter : Preferences :: FamilyConverter ) ]
if node . is_a? ( YAML :: Nodes :: Scalar )
property force_resolve : Socket :: Family = Socket :: Family :: UNSPEC # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
case node . value . downcase
property port : Int32 = 3000 # Port to listen for connections (overrided by command line argument)
when " ipv4 "
property host_binding : String = " 0.0.0.0 " # Host to bind (overrided by command line argument)
Socket :: Family :: INET
property pool_size : Int32 = 100 # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
when " ipv6 "
property admin_email : String = " omarroth@protonmail.com " # Email for bug reports
Socket :: Family :: INET6
else
@[ YAML :: Field ( converter : Preferences :: StringToCookies ) ]
Socket :: Family :: UNSPEC
property cookies : HTTP :: Cookies = HTTP :: Cookies . new # Saved cookies in "name1=value1; name2=value2..." format
end
property captcha_key : String ? = nil # Key for Anti-Captcha
else
node . raise " Expected scalar, not #{ node . class } "
end
end
end
module StringToCookies
def self . to_yaml ( value : HTTP :: Cookies , yaml : YAML :: Nodes :: Builder )
( value . map { | c | " #{ c . name } = #{ c . value } " } ) . join ( " ; " ) . to_yaml ( yaml )
end
def self . from_yaml ( ctx : YAML :: ParseContext , node : YAML :: Nodes :: Node ) : HTTP :: Cookies
unless node . is_a? ( YAML :: Nodes :: Scalar )
node . raise " Expected scalar, not #{ node . class } "
end
cookies = HTTP :: Cookies . new
node . value . split ( " ; " ) . each do | cookie |
next if cookie . strip . empty?
name , value = cookie . split ( " = " , 2 )
cookies << HTTP :: Cookie . new ( name . strip , value . strip )
end
cookies
end
end
def disabled? ( option )
def disabled? ( option )
case disabled = CONFIG . disable_proxy
case disabled = CONFIG . disable_proxy
@ -229,50 +110,16 @@ struct Config
return false
return false
end
end
end
end
YAML . mapping ( {
channel_threads : Int32 , # Number of threads to use for crawling videos from channels (for updating subscriptions)
feed_threads : Int32 , # Number of threads to use for updating feeds
db : DBConfig , # Database configuration
full_refresh : Bool , # Used for crawling channels: threads should check all videos uploaded by a channel
https_only : Bool ?, # Used to tell Invidious it is behind a proxy, so links to resources should be https://
hmac_key : String ?, # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
domain : String ?, # Domain to be used for links to resources on the site where an absolute URL is required
use_pubsub_feeds : { type : Bool | Int32 , default : false } , # Subscribe to channels using PubSubHubbub (requires domain, hmac_key)
captcha_enabled : { type : Bool , default : true } ,
login_enabled : { type : Bool , default : true } ,
registration_enabled : { type : Bool , default : true } ,
statistics_enabled : { type : Bool , default : false } ,
admins : { type : Array ( String ) , default : [ ] of String } ,
external_port : { type : Int32 ?, default : nil } ,
default_user_preferences : { type : Preferences ,
default : Preferences . new ( * ConfigPreferences . from_yaml ( " " ) . to_tuple ) ,
converter : ConfigPreferencesConverter ,
} ,
dmca_content : { type : Array ( String ) , default : [ ] of String } , # For compliance with DMCA, disables download widget using list of video IDs
check_tables : { type : Bool , default : false } , # Check table integrity, automatically try to add any missing columns, create tables, etc.
cache_annotations : { type : Bool , default : false } , # Cache annotations requested from IA, will not cache empty annotations or annotations that only contain cards
banner : { type : String ?, default : nil } , # Optional banner to be displayed along top of page for announcements, etc.
hsts : { type : Bool ?, default : true } , # Enables 'Strict-Transport-Security'. Ensure that `domain` and all subdomains are served securely
disable_proxy : { type : Bool ? | Array ( String ) ?, default : false } , # Disable proxying server-wide: options: 'dash', 'livestreams', 'downloads', 'local'
force_resolve : { type : Socket :: Family , default : Socket :: Family :: UNSPEC , converter : FamilyConverter } , # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
port : { type : Int32 , default : 3000 } , # Port to listen for connections (overrided by command line argument)
host_binding : { type : String , default : " 0.0.0.0 " } , # Host to bind (overrided by command line argument)
pool_size : { type : Int32 , default : 100 } , # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
admin_email : { type : String , default : " omarroth@protonmail.com " } , # Email for bug reports
cookies : { type : HTTP :: Cookies , default : HTTP :: Cookies . new , converter : StringToCookies } , # Saved cookies in "name1=value1; name2=value2..." format
captcha_key : { type : String ?, default : nil } , # Key for Anti-Captcha
} )
end end
struct DBConfig struct DBConfig
yaml_mapping ( {
include YAML :: Serializable
user : String ,
password : String ,
property user : String
host : String ,
property password : String
port : Int32 ,
property host : String
dbname : String ,
property port : Int32
} )
property dbname : String
end end
def login_req ( f_req ) def login_req ( f_req )
@ -365,7 +212,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
end
end
end
end
items << SearchVideo . new (
items << SearchVideo . new ( {
title : title ,
title : title ,
id : video_id ,
id : video_id ,
author : author ,
author : author ,
@ -377,8 +224,8 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
live_now : live_now ,
live_now : live_now ,
paid : paid ,
paid : paid ,
premium : premium ,
premium : premium ,
premiere_timestamp : premiere_timestamp
premiere_timestamp : premiere_timestamp ,
} )
elsif i = item [ " channelRenderer " ]?
elsif i = item [ " channelRenderer " ]?
author = i [ " title " ] [ " simpleText " ]? . try & . as_s || author_fallback || " "
author = i [ " title " ] [ " simpleText " ]? . try & . as_s || author_fallback || " "
author_id = i [ " channelId " ]? . try & . as_s || author_id_fallback || " "
author_id = i [ " channelId " ]? . try & . as_s || author_id_fallback || " "
@ -391,7 +238,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
video_count = i [ " videoCountText " ]? . try & . [ " runs " ] . as_a [ 0 ]? . try & . [ " text " ] . as_s . gsub ( / \ D / , " " ) . to_i || 0
video_count = i [ " videoCountText " ]? . try & . [ " runs " ] . as_a [ 0 ]? . try & . [ " text " ] . as_s . gsub ( / \ D / , " " ) . to_i || 0
description_html = i [ " descriptionSnippet " ]? . try { | t | parse_content ( t ) } || " "
description_html = i [ " descriptionSnippet " ]? . try { | t | parse_content ( t ) } || " "
items << SearchChannel . new (
items << SearchChannel . new ( {
author : author ,
author : author ,
ucid : author_id ,
ucid : author_id ,
author_thumbnail : author_thumbnail ,
author_thumbnail : author_thumbnail ,
@ -399,7 +246,7 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
video_count : video_count ,
video_count : video_count ,
description_html : description_html ,
description_html : description_html ,
auto_generated : auto_generated ,
auto_generated : auto_generated ,
} )
elsif i = item [ " gridPlaylistRenderer " ]?
elsif i = item [ " gridPlaylistRenderer " ]?
title = i [ " title " ] [ " runs " ] . as_a [ 0 ]? . try & . [ " text " ] . as_s || " "
title = i [ " title " ] [ " runs " ] . as_a [ 0 ]? . try & . [ " text " ] . as_s || " "
plid = i [ " playlistId " ]? . try & . as_s || " "
plid = i [ " playlistId " ]? . try & . as_s || " "
@ -407,15 +254,15 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
video_count = i [ " videoCountText " ] [ " runs " ] . as_a [ 0 ]? . try & . [ " text " ] . as_s . gsub ( / \ D / , " " ) . to_i || 0
video_count = i [ " videoCountText " ] [ " runs " ] . as_a [ 0 ]? . try & . [ " text " ] . as_s . gsub ( / \ D / , " " ) . to_i || 0
playlist_thumbnail = i [ " thumbnail " ] [ " thumbnails " ] [ 0 ]? . try & . [ " url " ]? . try & . as_s || " "
playlist_thumbnail = i [ " thumbnail " ] [ " thumbnails " ] [ 0 ]? . try & . [ " url " ]? . try & . as_s || " "
items << SearchPlaylist . new (
items << SearchPlaylist . new ( {
title : title ,
title : title ,
id : plid ,
id : plid ,
author : author_fallback || " " ,
author : author_fallback || " " ,
ucid : author_id_fallback || " " ,
ucid : author_id_fallback || " " ,
video_count : video_count ,
video_count : video_count ,
videos : [ ] of SearchPlaylistVideo ,
videos : [ ] of SearchPlaylistVideo ,
thumbnail : playlist_thumbnail
thumbnail : playlist_thumbnail ,
} )
elsif i = item [ " playlistRenderer " ]?
elsif i = item [ " playlistRenderer " ]?
title = i [ " title " ] [ " simpleText " ]? . try & . as_s || " "
title = i [ " title " ] [ " simpleText " ]? . try & . as_s || " "
plid = i [ " playlistId " ]? . try & . as_s || " "
plid = i [ " playlistId " ]? . try & . as_s || " "
@ -432,24 +279,24 @@ def extract_items(initial_data : Hash(String, JSON::Any), author_fallback : Stri
v_title = v [ " title " ] [ " simpleText " ]? . try & . as_s || " "
v_title = v [ " title " ] [ " simpleText " ]? . try & . as_s || " "
v_id = v [ " videoId " ]? . try & . as_s || " "
v_id = v [ " videoId " ]? . try & . as_s || " "
v_length_seconds = v [ " lengthText " ]? . try & . [ " simpleText " ]? . try { | t | decode_length_seconds ( t . as_s ) } || 0
v_length_seconds = v [ " lengthText " ]? . try & . [ " simpleText " ]? . try { | t | decode_length_seconds ( t . as_s ) } || 0
SearchPlaylistVideo . new (
SearchPlaylistVideo . new ( {
title : v_title ,
title : v_title ,
id : v_id ,
id : v_id ,
length_seconds : v_length_seconds
length_seconds : v_length_seconds ,
} )
end || [ ] of SearchPlaylistVideo
end || [ ] of SearchPlaylistVideo
# TODO: i["publishedTimeText"]?
# TODO: i["publishedTimeText"]?
items << SearchPlaylist . new (
items << SearchPlaylist . new ( {
title : title ,
title : title ,
id : plid ,
id : plid ,
author : author ,
author : author ,
ucid : author_id ,
ucid : author_id ,
video_count : video_count ,
video_count : video_count ,
videos : videos ,
videos : videos ,
thumbnail : playlist_thumbnail
thumbnail : playlist_thumbnail ,
} )
elsif i = item [ " radioRenderer " ]? # Mix
elsif i = item [ " radioRenderer " ]? # Mix
# TODO
# TODO
elsif i = item [ " showRenderer " ]? # Show
elsif i = item [ " showRenderer " ]? # Show
@ -465,6 +312,7 @@ end
def check_enum ( db , logger , enum_name , struct_type = nil ) def check_enum ( db , logger , enum_name , struct_type = nil )
return # TODO
return # TODO
if ! db . query_one? ( " SELECT true FROM pg_type WHERE typname = $1 " , enum_name , as : Bool )
if ! db . query_one? ( " SELECT true FROM pg_type WHERE typname = $1 " , enum_name , as : Bool )
logger . puts ( " CREATE TYPE #{ enum_name } " )
logger . puts ( " CREATE TYPE #{ enum_name } " )
@ -488,7 +336,7 @@ def check_table(db, logger, table_name, struct_type = nil)
return if ! struct_type
return if ! struct_type
struct_array = struct_type . t o_type_tuple
struct_array = struct_type . t ype_array
column_array = get_column_array ( db , table_name )
column_array = get_column_array ( db , table_name )
column_types = File . read ( " config/sql/ #{ table_name } .sql " ) . match ( / CREATE TABLE public \ . #{ table_name } \ n \ ((?<types>[ \ d \ D]*?) \ ); / )
column_types = File . read ( " config/sql/ #{ table_name } .sql " ) . match ( / CREATE TABLE public \ . #{ table_name } \ n \ ((?<types>[ \ d \ D]*?) \ ); / )
. try & . [ " types " ] . split ( " , " ) . map { | line | line . strip } . reject & . starts_with? ( " CONSTRAINT " )
. try & . [ " types " ] . split ( " , " ) . map { | line | line . strip } . reject & . starts_with? ( " CONSTRAINT " )
Omar Roth
4 years ago