syeopite
3 years ago
committed by
GitHub
commit
db7e7aae46
@ -0,0 +1,96 @@
|
|||||||
|
require "lsquic"
|
||||||
|
|
||||||
|
def add_yt_headers(request)
|
||||||
|
request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36"
|
||||||
|
request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
|
||||||
|
request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
|
||||||
|
request.headers["accept-language"] ||= "en-us,en;q=0.5"
|
||||||
|
return if request.resource.starts_with? "/sorry/index"
|
||||||
|
request.headers["x-youtube-client-name"] ||= "1"
|
||||||
|
request.headers["x-youtube-client-version"] ||= "2.20200609"
|
||||||
|
# Preserve original cookies and add new YT consent cookie for EU servers
|
||||||
|
request.headers["cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+"
|
||||||
|
if !CONFIG.cookies.empty?
|
||||||
|
request.headers["cookie"] = "#{(CONFIG.cookies.map { |c| "#{c.name}=#{c.value}" }).join("; ")}; #{request.headers["cookie"]?}"
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
struct YoutubeConnectionPool
|
||||||
|
property! url : URI
|
||||||
|
property! capacity : Int32
|
||||||
|
property! timeout : Float64
|
||||||
|
property pool : DB::Pool(QUIC::Client | HTTP::Client)
|
||||||
|
|
||||||
|
def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true)
|
||||||
|
@url = url
|
||||||
|
@pool = build_pool(use_quic)
|
||||||
|
end
|
||||||
|
|
||||||
|
def client(region = nil, &block)
|
||||||
|
if region
|
||||||
|
conn = make_client(url, region)
|
||||||
|
response = yield conn
|
||||||
|
else
|
||||||
|
conn = pool.checkout
|
||||||
|
begin
|
||||||
|
response = yield conn
|
||||||
|
rescue ex
|
||||||
|
conn.close
|
||||||
|
conn = QUIC::Client.new(url)
|
||||||
|
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
|
||||||
|
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
|
||||||
|
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
||||||
|
response = yield conn
|
||||||
|
ensure
|
||||||
|
pool.release(conn)
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
response
|
||||||
|
end
|
||||||
|
|
||||||
|
private def build_pool(use_quic)
|
||||||
|
DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do
|
||||||
|
if use_quic
|
||||||
|
conn = QUIC::Client.new(url)
|
||||||
|
else
|
||||||
|
conn = HTTP::Client.new(url)
|
||||||
|
end
|
||||||
|
conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET
|
||||||
|
conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC
|
||||||
|
conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
||||||
|
conn
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def make_client(url : URI, region = nil)
|
||||||
|
# TODO: Migrate any applicable endpoints to QUIC
|
||||||
|
client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure)
|
||||||
|
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
|
||||||
|
client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
|
||||||
|
client.read_timeout = 10.seconds
|
||||||
|
client.connect_timeout = 10.seconds
|
||||||
|
|
||||||
|
if region
|
||||||
|
PROXY_LIST[region]?.try &.sample(40).each do |proxy|
|
||||||
|
begin
|
||||||
|
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
|
||||||
|
client.set_proxy(proxy)
|
||||||
|
break
|
||||||
|
rescue ex
|
||||||
|
end
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
return client
|
||||||
|
end
|
||||||
|
|
||||||
|
def make_client(url : URI, region = nil, &block)
|
||||||
|
client = make_client(url, region)
|
||||||
|
begin
|
||||||
|
yield client
|
||||||
|
ensure
|
||||||
|
client.close
|
||||||
|
end
|
||||||
|
end
|
||||||
@ -0,0 +1,67 @@
|
|||||||
|
# Extracts text from InnerTube response
|
||||||
|
#
|
||||||
|
# InnerTube can package text in three different formats
|
||||||
|
# "runs": [
|
||||||
|
# {"text": "something"},
|
||||||
|
# {"text": "cont"},
|
||||||
|
# ...
|
||||||
|
# ]
|
||||||
|
#
|
||||||
|
# "SimpleText": "something"
|
||||||
|
#
|
||||||
|
# Or sometimes just none at all as with the data returned from
|
||||||
|
# category continuations.
|
||||||
|
#
|
||||||
|
# In order to facilitate calling this function with `#[]?`:
|
||||||
|
# A nil will be accepted. Of course, since nil cannot be parsed,
|
||||||
|
# another nil will be returned.
|
||||||
|
def extract_text(item : JSON::Any?) : String?
|
||||||
|
if item.nil?
|
||||||
|
return nil
|
||||||
|
end
|
||||||
|
|
||||||
|
if text_container = item["simpleText"]?
|
||||||
|
return text_container.as_s
|
||||||
|
elsif text_container = item["runs"]?
|
||||||
|
return text_container.as_a.map(&.["text"].as_s).join("")
|
||||||
|
else
|
||||||
|
nil
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
|
def extract_videos(initial_data : Hash(String, JSON::Any), author_fallback : String? = nil, author_id_fallback : String? = nil)
|
||||||
|
extracted = extract_items(initial_data, author_fallback, author_id_fallback)
|
||||||
|
|
||||||
|
target = [] of SearchItem
|
||||||
|
extracted.each do |i|
|
||||||
|
if i.is_a?(Category)
|
||||||
|
i.contents.each { |cate_i| target << cate_i if !cate_i.is_a? Video }
|
||||||
|
else
|
||||||
|
target << i
|
||||||
|
end
|
||||||
|
end
|
||||||
|
return target.select(&.is_a?(SearchVideo)).map(&.as(SearchVideo))
|
||||||
|
end
|
||||||
|
|
||||||
|
def extract_selected_tab(tabs)
|
||||||
|
# Extract the selected tab from the array of tabs Youtube returns
|
||||||
|
return selected_target = tabs.as_a.select(&.["tabRenderer"]?.try &.["selected"].as_bool)[0]["tabRenderer"]
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetch_continuation_token(items : Array(JSON::Any))
|
||||||
|
# Fetches the continuation token from an array of items
|
||||||
|
return items.last["continuationItemRenderer"]?
|
||||||
|
.try &.["continuationEndpoint"]["continuationCommand"]["token"].as_s
|
||||||
|
end
|
||||||
|
|
||||||
|
def fetch_continuation_token(initial_data : Hash(String, JSON::Any))
|
||||||
|
# Fetches the continuation token from initial data
|
||||||
|
if initial_data["onResponseReceivedActions"]?
|
||||||
|
continuation_items = initial_data["onResponseReceivedActions"][0]["appendContinuationItemsAction"]["continuationItems"]
|
||||||
|
else
|
||||||
|
tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"])
|
||||||
|
continuation_items = tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["gridRenderer"]["items"]
|
||||||
|
end
|
||||||
|
|
||||||
|
return fetch_continuation_token(continuation_items.as_a)
|
||||||
|
end
|
||||||
Loading…
Reference in New Issue