@ -1,5 +1,51 @@
struct PlaylistVideo struct PlaylistVideo
def to_json ( locale , config , kemal_config , json : JSON :: Builder )
def to_xml ( host_url , auto_generated , xml : XML :: Builder )
xml . element ( " entry " ) do
xml . element ( " id " ) { xml . text " yt:video: #{ self . id } " }
xml . element ( " yt:videoId " ) { xml . text self . id }
xml . element ( " yt:channelId " ) { xml . text self . ucid }
xml . element ( " title " ) { xml . text self . title }
xml . element ( " link " , rel : " alternate " , href : " #{ host_url } /watch?v= #{ self . id } " )
xml . element ( " author " ) do
if auto_generated
xml . element ( " name " ) { xml . text self . author }
xml . element ( " uri " ) { xml . text " #{ host_url } /channel/ #{ self . ucid } " }
else
xml . element ( " name " ) { xml . text author }
xml . element ( " uri " ) { xml . text " #{ host_url } /channel/ #{ ucid } " }
end
end
xml . element ( " content " , type : " xhtml " ) do
xml . element ( " div " , xmlns : " http://www.w3.org/1999/xhtml " ) do
xml . element ( " a " , href : " #{ host_url } /watch?v= #{ self . id } " ) do
xml . element ( " img " , src : " #{ host_url } /vi/ #{ self . id } /mqdefault.jpg " )
end
end
end
xml . element ( " published " ) { xml . text self . published . to_s ( " %Y-%m-%dT%H:%M:%S%:z " ) }
xml . element ( " media:group " ) do
xml . element ( " media:title " ) { xml . text self . title }
xml . element ( " media:thumbnail " , url : " #{ host_url } /vi/ #{ self . id } /mqdefault.jpg " ,
width : " 320 " , height : " 180 " )
end
end
end
def to_xml ( host_url , auto_generated , xml : XML :: Builder? = nil )
if xml
to_xml ( host_url , auto_generated , xml )
else
XML . build do | json |
to_xml ( host_url , auto_generated , xml )
end
end
end
def to_json ( locale , config , kemal_config , json : JSON :: Builder , index : Int32 ?)
json . object do
json . object do
json . field " title " , self . title
json . field " title " , self . title
json . field " videoId " , self . id
json . field " videoId " , self . id
@ -12,17 +58,23 @@ struct PlaylistVideo
generate_thumbnails ( json , self . id , config , kemal_config )
generate_thumbnails ( json , self . id , config , kemal_config )
end
end
json . field " index " , self . index
if index
json . field " index " , index
json . field " indexId " , self . index . to_u64 . to_s ( 16 ) . upcase
else
json . field " index " , self . index
end
json . field " lengthSeconds " , self . length_seconds
json . field " lengthSeconds " , self . length_seconds
end
end
end
end
def to_json ( locale , config , kemal_config , json : JSON :: Builder | Nil = nil )
def to_json ( locale , config , kemal_config , json : JSON :: Builder ? = nil , index : Int32 ? = nil )
if json
if json
to_json ( locale , config , kemal_config , json
to_json ( locale , config , kemal_config , json , index : index )
else
else
JSON . build do | json |
JSON . build do | json |
to_json ( locale , config , kemal_config , json
to_json ( locale , config , kemal_config , json , index : index )
end
end
end
end
end
end
@ -35,12 +87,66 @@ struct PlaylistVideo
length_seconds : Int32 ,
length_seconds : Int32 ,
published : Time ,
published : Time ,
plid : String ,
plid : String ,
index : Int 32 ,
index : Int 64 ,
live_now : Bool ,
live_now : Bool ,
} )
} )
end end
struct Playlist struct Playlist
def to_json ( offset , locale , config , kemal_config , json : JSON :: Builder , continuation : String ? = nil )
json . object do
json . field " type " , " playlist "
json . field " title " , self . title
json . field " playlistId " , self . id
json . field " playlistThumbnail " , self . thumbnail
json . field " author " , self . author
json . field " authorId " , self . ucid
json . field " authorUrl " , " /channel/ #{ self . ucid } "
json . field " authorThumbnails " do
json . array do
qualities = { 32 , 48 , 76 , 100 , 176 , 512 }
qualities . each do | quality |
json . object do
json . field " url " , self . author_thumbnail . not_nil! . gsub ( / = \ d+ / , " =s #{ quality } " )
json . field " width " , quality
json . field " height " , quality
end
end
end
end
json . field " description " , html_to_content ( self . description_html )
json . field " descriptionHtml " , self . description_html
json . field " videoCount " , self . video_count
json . field " viewCount " , self . views
json . field " updated " , self . updated . to_unix
json . field " isListed " , self . privacy . public?
json . field " videos " do
json . array do
videos = get_playlist_videos ( PG_DB , self , offset : offset , locale : locale , continuation : continuation )
videos . each_with_index do | video , index |
video . to_json ( locale , config , Kemal . config , json )
end
end
end
end
end
def to_json ( offset , locale , config , kemal_config , json : JSON :: Builder? = nil , continuation : String ? = nil )
if json
to_json ( offset , locale , config , kemal_config , json , continuation : continuation )
else
JSON . build do | json |
to_json ( offset , locale , config , kemal_config , json , continuation : continuation )
end
end
end
db_mapping ( {
db_mapping ( {
title : String ,
title : String ,
id : String ,
id : String ,
@ -53,57 +159,122 @@ struct Playlist
updated : Time ,
updated : Time ,
thumbnail : String ?,
thumbnail : String ?,
} )
} )
def privacy
PlaylistPrivacy :: Public
end
end end
def fetch_playlist_videos ( plid , page , video_count , continuation = nil , locale = nil ) enum PlaylistPrivacy
client = make_client ( YT_URL )
Public = 0
Unlisted = 1
Private = 2
end
if continuation
struct InvidiousPlaylist
html = client . get ( " /watch?v= #{ continuation } &list= #{ plid } &gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999 " )
def to_json ( offset , locale , config , kemal_config , json : JSON :: Builder , continuation : String ? = nil )
html = XML . parse_html ( html . body )
json . object do
json . field " type " , " invidiousPlaylist "
json . field " title " , self . title
json . field " playlistId " , self . id
index = html . xpath_node ( % q ( / /s pan [ @id = " playlist-current-index " ] ) ) . try & . content . to_i?
json . field " author " , self . author
if index
json . field " authorId " , self . ucid
index -= 1
json . field " authorUrl " , nil
json . field " authorThumbnails " , [ ] of String
json . field " description " , html_to_content ( self . description_html )
json . field " descriptionHtml " , self . description_html
json . field " videoCount " , self . video_count
json . field " viewCount " , self . views
json . field " updated " , self . updated . to_unix
json . field " isListed " , self . privacy . public?
json . field " videos " do
json . array do
videos = get_playlist_videos ( PG_DB , self , offset : offset , locale : locale , continuation : continuation )
videos . each_with_index do | video , index |
video . to_json ( locale , config , Kemal . config , json , offset + index )
end
end
end
end
end
index || = 0
else
index = ( page - 1 ) * 100
end
end
if video_count > 100
def to_json ( offset , locale , config , kemal_config , json : JSON :: Builder? = nil , continuation : String ? = nil )
url = produce_playlist_url ( plid , index )
if json
to_json ( offset , locale , config , kemal_config , json , continuation : continuation )
else
JSON . build do | json |
to_json ( offset , locale , config , kemal_config , json , continuation : continuation )
end
end
end
response = client . get ( url )
property thumbnail_id
response = JSON . parse ( response . body )
if ! response [ " content_html " ]? || response [ " content_html " ] . as_s . empty?
module PlaylistPrivacyConverter
raise translate ( locale , " Empty playlist " )
def self . from_rs ( rs )
return PlaylistPrivacy . parse ( String . new ( rs . read ( Slice ( UInt8 ) ) ) )
end
end
end
document = XML . parse_html ( response [ " content_html " ] . as_s )
db_mapping ( {
nodeset = document . xpath_nodes ( % q ( . / / tr [ contains ( @class , " pl-video " ) ] ) )
title : String ,
videos = extract_playlist ( plid , nodeset , index )
id : String ,
else
author : String ,
# Playlist has less than one page of videos, so subsequent pages will be empty
description : { type : String , default : " " } ,
if page > 1
video_count : Int32 ,
videos = [ ] of PlaylistVideo
created : Time ,
else
updated : Time ,
# Extract first page of videos
privacy : { type : PlaylistPrivacy , default : PlaylistPrivacy :: Private , converter : PlaylistPrivacyConverter } ,
response = client . get ( " /playlist?list= #{ plid } &gl=US&hl=en&disable_polymer=1 " )
index : Array ( Int64 ) ,
document = XML . parse_html ( response . body )
} )
nodeset = document . xpath_nodes ( % q ( . / / tr [ contains ( @class , " pl-video " ) ] ) )
videos = extract_playlist ( plid , nodeset , 0 )
def thumbnail
@thumbnail_id || = PG_DB . query_one? ( " SELECT id FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 1 " , self . id , self . index , as : String ) || " ----------- "
" /vi/ #{ @thumbnail_id } /mqdefault.jpg "
end
if continuation
def author_thumbnail
until videos [ 0 ] . id == continuation
nil
videos . shift
end
end
end
end
end
return videos
def ucid
nil
end
def views
0 _i64
end
def description_html
HTML . escape ( self . description ) . gsub ( " \n " , " <br> " )
end
end
def create_playlist ( db , title , privacy , user )
plid = " IVPL #{ Random :: Secure . urlsafe_base64 ( 24 ) [ 0 , 31 ] } "
playlist = InvidiousPlaylist . new (
title : title . byte_slice ( 0 , 150 ) ,
id : plid ,
author : user . email ,
description : " " , # Max 5000 characters
video_count : 0 ,
created : Time . utc ,
updated : Time . utc ,
privacy : privacy ,
index : [ ] of Int64 ,
)
playlist_array = playlist . to_a
args = arg_array ( playlist_array )
db . exec ( " INSERT INTO playlists VALUES ( #{ args } ) " , args : playlist_array )
return playlist
end end
def extract_playlist ( plid , nodeset , index ) def extract_playlist ( plid , nodeset , index )
@ -144,7 +315,7 @@ def extract_playlist(plid, nodeset, index)
length_seconds : length_seconds ,
length_seconds : length_seconds ,
published : Time . utc ,
published : Time . utc ,
plid : plid ,
plid : plid ,
index : index + offset ,
index : ( index + offset ) . to_i64 ,
live_now : live_now
live_now : live_now
)
)
end
end
@ -200,6 +371,18 @@ def produce_playlist_url(id, index)
return url
return url
end end
def get_playlist ( db , plid , locale , refresh = true , force_refresh = false )
if plid . starts_with? " IV "
if playlist = db . query_one? ( " SELECT * FROM playlists WHERE id = $1 " , plid , as : InvidiousPlaylist )
return playlist
else
raise " Playlist does not exist. "
end
else
return fetch_playlist ( plid , locale )
end
end
def fetch_playlist ( plid , locale ) def fetch_playlist ( plid , locale )
client = make_client ( YT_URL )
client = make_client ( YT_URL )
@ -261,6 +444,59 @@ def fetch_playlist(plid, locale)
return playlist
return playlist
end end
def get_playlist_videos ( db , playlist , offset , locale = nil , continuation = nil )
if playlist . is_a? InvidiousPlaylist
if ! offset
index = PG_DB . query_one? ( " SELECT index FROM playlist_videos WHERE plid = $1 AND id = $2 LIMIT 1 " , playlist . id , continuation , as : Int64 )
offset = playlist . index . index ( index ) || 0
end
db . query_all ( " SELECT * FROM playlist_videos WHERE plid = $1 ORDER BY array_position($2, index) LIMIT 100 OFFSET $3 " , playlist . id , playlist . index , offset , as : PlaylistVideo )
else
fetch_playlist_videos ( playlist . id , playlist . video_count , offset , locale , continuation )
end
end
def fetch_playlist_videos ( plid , video_count , offset = 0 , locale = nil , continuation = nil )
client = make_client ( YT_URL )
if continuation
html = client . get ( " /watch?v= #{ continuation } &list= #{ plid } &gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999 " )
html = XML . parse_html ( html . body )
index = html . xpath_node ( % q ( / /s pan [ @id = " playlist-current-index " ] ) ) . try & . content . to_i? . try & . - 1
offset = index || offset
end
if video_count > 100
url = produce_playlist_url ( plid , offset )
response = client . get ( url )
response = JSON . parse ( response . body )
if ! response [ " content_html " ]? || response [ " content_html " ] . as_s . empty?
raise translate ( locale , " Empty playlist " )
end
document = XML . parse_html ( response [ " content_html " ] . as_s )
nodeset = document . xpath_nodes ( % q ( . / / tr [ contains ( @class , " pl-video " ) ] ) )
videos = extract_playlist ( plid , nodeset , offset )
elsif offset > 100
return [ ] of PlaylistVideo
else # Extract first page of videos
response = client . get ( " /playlist?list= #{ plid } &gl=US&hl=en&disable_polymer=1 " )
document = XML . parse_html ( response . body )
nodeset = document . xpath_nodes ( % q ( . / / tr [ contains ( @class , " pl-video " ) ] ) )
videos = extract_playlist ( plid , nodeset , 0 )
end
until videos . empty? || videos [ 0 ] . index == offset
videos . shift
end
return videos
end
def template_playlist ( playlist ) def template_playlist ( playlist )
html = << - END_HTML
html = << - END_HTML
< h3 >
< h3 >
Omar Roth
5 years ago
committed by