|
|
@ -25,15 +25,15 @@ require "./helpers"
|
|
|
|
|
|
|
|
|
|
|
|
CONFIG = Config.from_yaml(File.read("config/config.yml"))
|
|
|
|
CONFIG = Config.from_yaml(File.read("config/config.yml"))
|
|
|
|
|
|
|
|
|
|
|
|
threads = CONFIG.threads
|
|
|
|
crawl_threads = CONFIG.crawl_threads
|
|
|
|
channel_threads = CONFIG.channel_threads
|
|
|
|
channel_threads = CONFIG.channel_threads
|
|
|
|
video_threads = CONFIG.video_threads
|
|
|
|
video_threads = CONFIG.video_threads
|
|
|
|
|
|
|
|
|
|
|
|
Kemal.config.extra_options do |parser|
|
|
|
|
Kemal.config.extra_options do |parser|
|
|
|
|
parser.banner = "Usage: invidious [arguments]"
|
|
|
|
parser.banner = "Usage: invidious [arguments]"
|
|
|
|
parser.on("-t THREADS", "--youtube-threads=THREADS", "Number of threads for crawling (default: #{threads})") do |number|
|
|
|
|
parser.on("-t THREADS", "--crawl-threads=THREADS", "Number of threads for crawling (default: #{crawl_threads})") do |number|
|
|
|
|
begin
|
|
|
|
begin
|
|
|
|
threads = number.to_i
|
|
|
|
crawl_threads = number.to_i
|
|
|
|
rescue ex
|
|
|
|
rescue ex
|
|
|
|
puts "THREADS must be integer"
|
|
|
|
puts "THREADS must be integer"
|
|
|
|
exit
|
|
|
|
exit
|
|
|
@ -73,7 +73,7 @@ YT_URL = URI.parse("https://www.youtube.com")
|
|
|
|
REDDIT_URL = URI.parse("https://api.reddit.com")
|
|
|
|
REDDIT_URL = URI.parse("https://api.reddit.com")
|
|
|
|
LOGIN_URL = URI.parse("https://accounts.google.com")
|
|
|
|
LOGIN_URL = URI.parse("https://accounts.google.com")
|
|
|
|
|
|
|
|
|
|
|
|
threads.times do
|
|
|
|
crawl_threads.times do
|
|
|
|
spawn do
|
|
|
|
spawn do
|
|
|
|
ids = Deque(String).new
|
|
|
|
ids = Deque(String).new
|
|
|
|
random = Random.new
|
|
|
|
random = Random.new
|
|
|
|