|
|
|
@ -44,27 +44,46 @@ def crawl_videos(db)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def refresh_channels(db, limit = 0, offset = 0)
|
|
|
|
|
def refresh_channels(db, max_threads = 1)
|
|
|
|
|
max_channel = Channel(Int32).new
|
|
|
|
|
|
|
|
|
|
spawn do
|
|
|
|
|
max_threads = max_channel.receive
|
|
|
|
|
active_threads = 0
|
|
|
|
|
active_channel = Channel(Bool).new
|
|
|
|
|
|
|
|
|
|
loop do
|
|
|
|
|
db.query("SELECT id FROM channels ORDER BY updated limit $1 offset $2", limit, offset) do |rs|
|
|
|
|
|
db.query("SELECT id FROM channels ORDER BY updated") do |rs|
|
|
|
|
|
rs.each do
|
|
|
|
|
client = make_client(YT_URL)
|
|
|
|
|
id = rs.read(String)
|
|
|
|
|
|
|
|
|
|
if active_threads >= max_threads
|
|
|
|
|
if active_channel.receive
|
|
|
|
|
active_threads -= 1
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
active_threads += 1
|
|
|
|
|
spawn do
|
|
|
|
|
begin
|
|
|
|
|
id = rs.read(String)
|
|
|
|
|
client = make_client(YT_URL)
|
|
|
|
|
channel = fetch_channel(id, client, db, false)
|
|
|
|
|
|
|
|
|
|
db.exec("UPDATE channels SET updated = $1 WHERE id = $2", Time.now, id)
|
|
|
|
|
rescue ex
|
|
|
|
|
STDOUT << id << " : " << ex.message << "\n"
|
|
|
|
|
next
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
active_channel.send(true)
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
Fiber.yield
|
|
|
|
|
end
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
max_channel.send(max_threads)
|
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
def refresh_videos(db)
|
|
|
|
|
loop do
|
|
|
|
|
db.query("SELECT id FROM videos ORDER BY updated") do |rs|
|
|
|
|
|