Use the new youtube api for comments (#2217)

* use the new youtube api for comments
* remove PG_DB & action parameter + allow force region
* support new comments data with onResponseReceivedEndpoints
pull/2323/head
Émilien Devos 3 years ago committed by GitHub
parent 2fdb2c7c9a
commit 88c5e3b6fa
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23

@ -1873,9 +1873,6 @@ get "/api/v1/comments/:id" do |env|
format = env.params.query["format"]? format = env.params.query["format"]?
format ||= "json" format ||= "json"
action = env.params.query["action"]?
action ||= "action_get_comments"
continuation = env.params.query["continuation"]? continuation = env.params.query["continuation"]?
sort_by = env.params.query["sort_by"]?.try &.downcase sort_by = env.params.query["sort_by"]?.try &.downcase
@ -1883,7 +1880,7 @@ get "/api/v1/comments/:id" do |env|
sort_by ||= "top" sort_by ||= "top"
begin begin
comments = fetch_youtube_comments(id, PG_DB, continuation, format, locale, thin_mode, region, sort_by: sort_by, action: action) comments = fetch_youtube_comments(id, continuation, format, locale, thin_mode, region, sort_by: sort_by)
rescue ex rescue ex
next error_json(500, ex) next error_json(500, ex)
end end

@ -56,10 +56,7 @@ class RedditListing
property modhash : String property modhash : String
end end
def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, sort_by = "top", action = "action_get_comments") def fetch_youtube_comments(id, cursor, format, locale, thin_mode, region, sort_by = "top")
video = get_video(id, db, region: region)
session_token = video.session_token
case cursor case cursor
when nil, "" when nil, ""
ctoken = produce_comment_continuation(id, cursor: "", sort_by: sort_by) ctoken = produce_comment_continuation(id, cursor: "", sort_by: sort_by)
@ -71,43 +68,41 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
ctoken = cursor ctoken = cursor
end end
if !session_token client_config = YoutubeAPI::ClientConfig.new(region: region)
if format == "json" response = YoutubeAPI.next(continuation: ctoken, client_config: client_config)
return {"comments" => [] of String}.to_json
if response["continuationContents"]?
response = response["continuationContents"]
if response["commentRepliesContinuation"]?
body = response["commentRepliesContinuation"]
else else
return {"contentHtml" => "", "commentCount" => 0}.to_json body = response["itemSectionContinuation"]
end
contents = body["contents"]?
header = body["header"]?
if body["continuations"]?
moreRepliesContinuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s
end
elsif response["onResponseReceivedEndpoints"]?
onResponseReceivedEndpoints = response["onResponseReceivedEndpoints"]
onResponseReceivedEndpoints.as_a.each do |item|
case item["reloadContinuationItemsCommand"]["slot"]
when "RELOAD_CONTINUATION_SLOT_HEADER"
header = item["reloadContinuationItemsCommand"]["continuationItems"][0]
when "RELOAD_CONTINUATION_SLOT_BODY"
contents = item["reloadContinuationItemsCommand"]["continuationItems"]
contents.as_a.reject! do |item|
if item["continuationItemRenderer"]?
moreRepliesContinuation = item["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s
true
end end
end end
post_req = {
page_token: ctoken,
session_token: session_token,
}
headers = HTTP::Headers{
"cookie" => video.cookie,
}
response = YT_POOL.client(region, &.post("/comment_service_ajax?#{action}=1&hl=en&gl=US&pbj=1", headers, form: post_req))
response = JSON.parse(response.body)
# For some reason youtube puts it in an array for comment_replies but otherwise it's the same
if action == "action_get_comment_replies"
response = response[1]
end end
if !response["response"]["continuationContents"]?
raise InfoException.new("Could not fetch comments")
end end
response = response["response"]["continuationContents"]
if response["commentRepliesContinuation"]?
body = response["commentRepliesContinuation"]
else else
body = response["itemSectionContinuation"] raise InfoException.new("Could not fetch comments")
end end
contents = body["contents"]?
if !contents if !contents
if format == "json" if format == "json"
return {"comments" => [] of String}.to_json return {"comments" => [] of String}.to_json
@ -118,11 +113,10 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
response = JSON.build do |json| response = JSON.build do |json|
json.object do json.object do
if body["header"]? if header
count_text = body["header"]["commentsHeaderRenderer"]["countText"] count_text = header["commentsHeaderRenderer"]["countText"]
comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?) comment_count = (count_text["simpleText"]? || count_text["runs"]?.try &.[0]?.try &.["text"]?)
.try &.as_s.gsub(/\D/, "").to_i? || 0 .try &.as_s.gsub(/\D/, "").to_i? || 0
json.field "commentCount", comment_count json.field "commentCount", comment_count
end end
@ -211,7 +205,11 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
reply_count = 1 reply_count = 1
end end
if node_replies["continuations"]?
continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s continuation = node_replies["continuations"]?.try &.as_a[0]["nextContinuationData"]["continuation"].as_s
elsif node_replies["contents"]?
continuation = node_replies["contents"]?.try &.as_a[0]["continuationItemRenderer"]["continuationEndpoint"]["continuationCommand"]["token"].as_s
end
continuation ||= "" continuation ||= ""
json.field "replies" do json.field "replies" do
@ -226,16 +224,15 @@ def fetch_youtube_comments(id, db, cursor, format, locale, thin_mode, region, so
end end
end end
if body["continuations"]? if moreRepliesContinuation
continuation = body["continuations"][0]["nextContinuationData"]["continuation"].as_s json.field "continuation", moreRepliesContinuation
json.field "continuation", continuation
end end
end end
end end
if format == "html" if format == "html"
response = JSON.parse(response) response = JSON.parse(response)
content_html = template_youtube_comments(response, locale, thin_mode, action == "action_get_comment_replies") content_html = template_youtube_comments(response, locale, thin_mode)
response = JSON.build do |json| response = JSON.build do |json|
json.object do json.object do

@ -92,7 +92,7 @@ module Invidious::Routes::Watch
if source == "youtube" if source == "youtube"
begin begin
comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
rescue ex rescue ex
if preferences.comments[1] == "reddit" if preferences.comments[1] == "reddit"
comments, reddit_thread = fetch_reddit_comments(id) comments, reddit_thread = fetch_reddit_comments(id)
@ -111,12 +111,12 @@ module Invidious::Routes::Watch
comment_html = replace_links(comment_html) comment_html = replace_links(comment_html)
rescue ex rescue ex
if preferences.comments[1] == "youtube" if preferences.comments[1] == "youtube"
comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end end
end end
end end
else else
comment_html = JSON.parse(fetch_youtube_comments(id, PG_DB, nil, "html", locale, preferences.thin_mode, region))["contentHtml"] comment_html = JSON.parse(fetch_youtube_comments(id, nil, "html", locale, preferences.thin_mode, region))["contentHtml"]
end end
comment_html ||= "" comment_html ||= ""

Loading…
Cancel
Save