|
|
@ -48,10 +48,7 @@ CONTEXT.add_options(
|
|
|
|
OpenSSL::SSL::Options::NO_SSL_V3
|
|
|
|
OpenSSL::SSL::Options::NO_SSL_V3
|
|
|
|
)
|
|
|
|
)
|
|
|
|
pool = Deque.new((threads * 1.2 + 1).to_i) do
|
|
|
|
pool = Deque.new((threads * 1.2 + 1).to_i) do
|
|
|
|
client = HTTP::Client.new(URL, CONTEXT)
|
|
|
|
make_client(URL, CONTEXT)
|
|
|
|
client.read_timeout = 10.seconds
|
|
|
|
|
|
|
|
client.connect_timeout = 10.seconds
|
|
|
|
|
|
|
|
client
|
|
|
|
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
# Refresh pool by crawling YT
|
|
|
|
# Refresh pool by crawling YT
|
|
|
@ -74,10 +71,7 @@ threads.times do
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
if rand(300) < 1
|
|
|
|
if rand(300) < 1
|
|
|
|
client = HTTP::Client.new(URL, CONTEXT)
|
|
|
|
pool << make_client(URL, CONTEXT)
|
|
|
|
client.read_timeout = 10.seconds
|
|
|
|
|
|
|
|
client.connect_timeout = 10.seconds
|
|
|
|
|
|
|
|
pool << client
|
|
|
|
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
time = Time.now
|
|
|
|
time = Time.now
|
|
|
@ -87,10 +81,7 @@ threads.times do
|
|
|
|
video = get_video(id, client, PG_DB)
|
|
|
|
video = get_video(id, client, PG_DB)
|
|
|
|
rescue ex
|
|
|
|
rescue ex
|
|
|
|
io << id << " : " << ex << "\n"
|
|
|
|
io << id << " : " << ex << "\n"
|
|
|
|
client = HTTP::Client.new(URL, CONTEXT)
|
|
|
|
pool << make_client(URL, CONTEXT)
|
|
|
|
client.read_timeout = 10.seconds
|
|
|
|
|
|
|
|
client.connect_timeout = 10.seconds
|
|
|
|
|
|
|
|
pool << client
|
|
|
|
|
|
|
|
next
|
|
|
|
next
|
|
|
|
ensure
|
|
|
|
ensure
|
|
|
|
ids.delete(id)
|
|
|
|
ids.delete(id)
|
|
|
|