Add fix for channel job

pull/91/head
Omar Roth 6 years ago
parent ca2320f17f
commit 923f9a716b

@ -83,9 +83,13 @@ crawl_threads.times do
end end
end end
total_channels = PG_DB.query_one("SELECT count(*) FROM channels", as: Int64)
channel_threads.times do |i| channel_threads.times do |i|
limit = total_channels / channel_threads
offset = limit.not_nil! * i
spawn do spawn do
refresh_channels(PG_DB) refresh_channels(PG_DB, limit, offset)
end end
end end

@ -44,9 +44,9 @@ def crawl_videos(db)
end end
end end
def refresh_channels(db) def refresh_channels(db, limit = 0, offset = 0)
loop do loop do
db.query("SELECT id FROM channels ORDER BY updated") do |rs| db.query("SELECT id FROM channels ORDER BY updated limit $1 offset $2", limit, offset) do |rs|
rs.each do rs.each do
client = make_client(YT_URL) client = make_client(YT_URL)

Loading…
Cancel
Save