helix
/
invidious
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move import logic to its own module

Browse Source
pull/2871/head
Samantaz Fox 3 years ago
parent ef8dc7272b
commit 2bbd424fce
No known key found for this signature in database
GPG Key ID: F42821059186176E
3 changed files with 203 additions and 136 deletions
Show Stats Download Patch File Download Diff File

4
spec/invidious/user/imports_spec.cr
Unescape Escape View File

@ -25,9 +25,9 @@ def csv_sample
CSV
end
Spectator.describe "Invidious::User::Imports" do
Spectator.describe Invidious::User::Import do
it "imports CSV" do
subscriptions = parse_subscription_export_csv(csv_sample)
subscriptions = Invidious::User::Import.parse_subscription_export_csv(csv_sample)
expect(subscriptions).to be_an(Array(String))
expect(subscriptions.size).to eq(13)

142
src/invidious/routes/preferences.cr
Unescape Escape View File

@ -321,149 +321,27 @@ module Invidious::Routes::PreferencesRoute
# TODO: Unify into single import based on content-type
case part.name
when "import_invidious"
body = JSON.parse(body)
if body["subscriptions"]?
user.subscriptions += body["subscriptions"].as_a.map(&.as_s)
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
end
if body["watch_history"]?
user.watched += body["watch_history"].as_a.map(&.as_s)
user.watched.uniq!
Invidious::Database::Users.update_watch_history(user)
end
if body["preferences"]?
user.preferences = Preferences.from_json(body["preferences"].to_json)
Invidious::Database::Users.update_preferences(user)
end
if playlists = body["playlists"]?.try &.as_a?
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
next if !title
next if !description
next if !privacy
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
video_id = video_id.try &.as_s?
next if !video_id
begin
video = get_video(video_id)
rescue ex
next
end
playlist_video = PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
length_seconds: video.length_seconds,
published: video.published,
plid: playlist.id,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX),
})
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
end
end
end
Invidious::User::Import.from_invidious(user, body)
when "import_youtube"
filename = part.filename || ""
extension = filename.split(".").last
success = Invidious::User::Import.from_youtube(user, body, filename, type)
if extension == "xml" || type == "application/xml" || type == "text/xml"
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
user.subscriptions += subscriptions.as_a.compact_map do |entry|
entry["snippet"]["resourceId"]["channelId"].as_s
end
elsif extension == "csv" || type == "text/csv"
subscriptions = parse_subscription_export_csv(body)
user.subscriptions += subscriptions
else
if !success
haltf(env, status_code: 415,
response: error_template(415, "Invalid subscription file uploaded")
)
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
when "import_freetube"
user.subscriptions += body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/).map do |md|
md["channel_id"]
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
Invidious::User::Import.from_freetube(user, body)
when "import_newpipe_subscriptions"
body = JSON.parse(body)
user.subscriptions += body["subscriptions"].as_a.compact_map do |channel|
if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
next match["channel"]
elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
response = YT_POOL.client &.get("/user/#{match["user"]}?disable_polymer=1&hl=en&gl=US")
html = XML.parse_html(response.body)
ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1]
next ucid if ucid
end
nil
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
Invidious::User::Import.from_newpipe_subs(user, body)
when "import_newpipe"
Compress::Zip::Reader.open(IO::Memory.new(body)) do |file|
file.each_entry do |entry|
if entry.filename == "newpipe.db"
tempfile = File.tempfile(".db")
File.write(tempfile.path, entry.io.gets_to_end)
db = DB.open("sqlite3://" + tempfile.path)
user.watched += db.query_all("SELECT url FROM streams", as: String).map(&.lchop("https://www.youtube.com/watch?v="))
user.watched.uniq!
Invidious::Database::Users.update_watch_history(user)
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/"))
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
success = Invidious::User::Import.from_newpipe(user, body)
Invidious::Database::Users.update_subscriptions(user)
db.close
tempfile.delete
end
end
if !success
haltf(env, status_code: 415,
response: error_template(415, "Uploaded file is too large")
)
end
else nil # Ignore
end

189
src/invidious/user/imports.cr
Unescape Escape View File

@ -29,5 +29,194 @@ struct Invidious::User
return subscriptions
end
# -------------------
# Invidious
# -------------------
# Import from another invidious account
def from_invidious(user : User, body : String)
data = JSON.parse(body)
if data["subscriptions"]?
user.subscriptions += data["subscriptions"].as_a.map(&.as_s)
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
end
if data["watch_history"]?
user.watched += data["watch_history"].as_a.map(&.as_s)
user.watched.uniq!
Invidious::Database::Users.update_watch_history(user)
end
if data["preferences"]?
user.preferences = Preferences.from_json(data["preferences"].to_json)
Invidious::Database::Users.update_preferences(user)
end
if playlists = data["playlists"]?.try &.as_a?
playlists.each do |item|
title = item["title"]?.try &.as_s?.try &.delete("<>")
description = item["description"]?.try &.as_s?.try &.delete("\r")
privacy = item["privacy"]?.try &.as_s?.try { |privacy| PlaylistPrivacy.parse? privacy }
next if !title
next if !description
next if !privacy
playlist = create_playlist(title, privacy, user)
Invidious::Database::Playlists.update_description(playlist.id, description)
videos = item["videos"]?.try &.as_a?.try &.each_with_index do |video_id, idx|
raise InfoException.new("Playlist cannot have more than 500 videos") if idx > 500
video_id = video_id.try &.as_s?
next if !video_id
begin
video = get_video(video_id)
rescue ex
next
end
playlist_video = PlaylistVideo.new({
title: video.title,
id: video.id,
author: video.author,
ucid: video.ucid,
length_seconds: video.length_seconds,
published: video.published,
plid: playlist.id,
live_now: video.live_now,
index: Random::Secure.rand(0_i64..Int64::MAX),
})
Invidious::Database::PlaylistVideos.insert(playlist_video)
Invidious::Database::Playlists.update_video_added(playlist.id, playlist_video.index)
end
end
end
end
# -------------------
# Youtube
# -------------------
# Import subscribed channels from Youtube
# Returns success status
def from_youtube(user : User, body : String, filename : String, type : String) : Bool
extension = filename.split(".").last
if extension == "xml" || type == "application/xml" || type == "text/xml"
subscriptions = XML.parse(body)
user.subscriptions += subscriptions.xpath_nodes(%q(//outline[@type="rss"])).map do |channel|
channel["xmlUrl"].match(/UC[a-zA-Z0-9_-]{22}/).not_nil![0]
end
elsif extension == "json" || type == "application/json"
subscriptions = JSON.parse(body)
user.subscriptions += subscriptions.as_a.compact_map do |entry|
entry["snippet"]["resourceId"]["channelId"].as_s
end
elsif extension == "csv" || type == "text/csv"
subscriptions = parse_subscription_export_csv(body)
user.subscriptions += subscriptions
else
return false
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
return true
end
# -------------------
# Freetube
# -------------------
def from_freetube(user : User, body : String)
matches = body.scan(/"channelId":"(?<channel_id>[a-zA-Z0-9_-]{24})"/)
user.subscriptions += matches.map(&.["channel_id"])
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
end
# -------------------
# Newpipe
# -------------------
def from_newpipe_subs(user : User, body : String)
data = JSON.parse(body)
user.subscriptions += data["subscriptions"].as_a.compact_map do |channel|
if match = channel["url"].as_s.match(/\/channel\/(?<channel>UC[a-zA-Z0-9_-]{22})/)
next match["channel"]
elsif match = channel["url"].as_s.match(/\/user\/(?<user>.+)/)
# Resolve URL using the API
resolved_url = YoutubeAPI.resolve_url("https://www.youtube.com/user/#{match["user"]}")
ucid = resolved_url.dig?("endpoint", "browseEndpoint", "browseId")
next ucid.as_s if ucid
end
nil
end
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
end
def from_newpipe(user : User, body : String) : Bool
io = IO::Memory.new(body)
Compress::Zip::File.open(io) do |file|
file.entries.each do |entry|
entry.open do |file_io|
# Ensure max size of 4MB
io_sized = IO::Sized.new(file_io, 0x400000)
next if entry.filename != "newpipe.db"
tempfile = File.tempfile(".db")
begin
File.write(tempfile.path, io_sized.gets_to_end)
rescue
return false
end
db = DB.open("sqlite3://" + tempfile.path)
user.watched += db.query_all("SELECT url FROM streams", as: String)
.map(&.lchop("https://www.youtube.com/watch?v="))
user.watched.uniq!
Invidious::Database::Users.update_watch_history(user)
user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String)
.map(&.lchop("https://www.youtube.com/channel/"))
user.subscriptions.uniq!
user.subscriptions = get_batch_channels(user.subscriptions)
Invidious::Database::Users.update_subscriptions(user)
db.close
tempfile.delete
end
end
end
# Success!
return true
end
end # module
end

Write Preview
Loading…
Cancel
Save