|
|
@ -1,4 +1,4 @@
|
|
|
|
def crawl_videos(db)
|
|
|
|
def crawl_videos(db, logger)
|
|
|
|
ids = Deque(String).new
|
|
|
|
ids = Deque(String).new
|
|
|
|
random = Random.new
|
|
|
|
random = Random.new
|
|
|
|
|
|
|
|
|
|
|
@ -46,7 +46,7 @@ def crawl_videos(db)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
def refresh_channels(db, max_threads = 1, full_refresh = false)
|
|
|
|
def refresh_channels(db, logger, max_threads = 1, full_refresh = false)
|
|
|
|
max_channel = Channel(Int32).new
|
|
|
|
max_channel = Channel(Int32).new
|
|
|
|
|
|
|
|
|
|
|
|
spawn do
|
|
|
|
spawn do
|
|
|
@ -86,7 +86,7 @@ def refresh_channels(db, max_threads = 1, full_refresh = false)
|
|
|
|
max_channel.send(max_threads)
|
|
|
|
max_channel.send(max_threads)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
def refresh_videos(db)
|
|
|
|
def refresh_videos(db, logger)
|
|
|
|
loop do
|
|
|
|
loop do
|
|
|
|
db.query("SELECT id FROM videos ORDER BY updated") do |rs|
|
|
|
|
db.query("SELECT id FROM videos ORDER BY updated") do |rs|
|
|
|
|
rs.each do
|
|
|
|
rs.each do
|
|
|
@ -104,7 +104,7 @@ def refresh_videos(db)
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
|
|
|
|
def refresh_feeds(db, max_threads = 1)
|
|
|
|
def refresh_feeds(db, logger, max_threads = 1)
|
|
|
|
max_channel = Channel(Int32).new
|
|
|
|
max_channel = Channel(Int32).new
|
|
|
|
|
|
|
|
|
|
|
|
spawn do
|
|
|
|
spawn do
|
|
|
|