Try to speed up find_working_proxies

src/invidious.cr

@@ -107,14 +107,14 @@ end
 
 proxies = {} of String => Array({ip: String, port: Int32})
 if CONFIG.geo_bypass
-spawn do
+  spawn do
-  find_working_proxies(BYPASS_REGIONS) do |region, list|
+    find_working_proxies(BYPASS_REGIONS) do |region, list|
-    if !list.empty?
+      if !list.empty?
-      proxies[region] = list
+        proxies[region] = list
+      end
     end
   end
 end
-end
 
 before_all do |env|
   env.response.headers["X-XSS-Protection"] = "1; mode=block;"
@@ -1876,6 +1876,8 @@ get "/api/v1/comments/:id" do |env|
 
       proxies.each do |region, list|
         spawn do
+          proxy_html = %(<meta itemprop="regionsAllowed" content="">)
+
           list.each do |proxy|
             begin
               proxy_client = HTTPClient.new(YT_URL)
@@ -1886,10 +1888,10 @@ get "/api/v1/comments/:id" do |env|
               proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
               proxy_client.set_proxy(proxy)
 
-              proxy_html = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
+              response = proxy_client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
               proxy_headers = HTTP::Headers.new
-              proxy_headers["cookie"] = proxy_html.cookies.add_request_headers(headers)["cookie"]
+              proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
-              proxy_html = proxy_html.body
+              proxy_html = response.body
 
               if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
                 bypass_channel.send(nil)
@@ -1901,6 +1903,11 @@ get "/api/v1/comments/:id" do |env|
             rescue ex
             end
           end
+
+          # If none of the proxies we tried returned a valid response
+          if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
+            bypass_channel.send(nil)
+          end
         end
       end
 

src/invidious/helpers/proxy.cr

@@ -93,6 +93,25 @@ def get_proxies(country_code = "US")
   return get_nova_proxies(country_code)
 end
 
+def filter_proxies(proxies)
+  proxies.select! do |proxy|
+    begin
+      client = HTTPClient.new(YT_URL)
+      client.read_timeout = 10.seconds
+      client.connect_timeout = 10.seconds
+
+      proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
+      client.set_proxy(proxy)
+
+      client.head("/").status_code == 200
+    rescue ex
+      false
+    end
+  end
+
+  return proxies
+end
+
 def get_nova_proxies(country_code = "US")
   country_code = country_code.downcase
   client = HTTP::Client.new(URI.parse("https://www.proxynova.com"))
@@ -127,7 +146,7 @@ def get_nova_proxies(country_code = "US")
     proxies << {ip: ip, port: port, score: score}
   end
 
-  proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse
+  # proxies = proxies.sort_by { |proxy| proxy[:score] }.reverse
   return proxies
 end
 

src/invidious/jobs.cr

@@ -156,39 +156,14 @@ def update_decrypt_function
 end
 
 def find_working_proxies(regions)
-  proxy_channel = Channel({String, Array({ip: String, port: Int32})}).new
+  loop do
-
+    regions.each do |region|
-  regions.each do |region|
+      proxies = get_proxies(region).first(20)
-    spawn do
+      proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
-      loop do
+      # proxies = filter_proxies(proxies)
-        begin
-          proxies = get_proxies(region).first(20)
-        rescue ex
-          next proxy_channel.send({region, Array({ip: String, port: Int32}).new})
-        end
-
-        proxies.select! do |proxy|
-          begin
-            client = HTTPClient.new(YT_URL)
-            client.read_timeout = 10.seconds
-            client.connect_timeout = 10.seconds
-
-            proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
-            client.set_proxy(proxy)
-
-            client.get("/").status_code == 200
-          rescue ex
-            false
-          end
-        end
-        proxies = proxies.map { |proxy| {ip: proxy[:ip], port: proxy[:port]} }
 
-        proxy_channel.send({region, proxies})
+      yield region, proxies
-      end
+      Fiber.yield
     end
   end
-
-  loop do
-    yield proxy_channel.receive
-  end
 end

src/invidious/videos.cr

@@ -543,6 +543,10 @@ def fetch_video(id, proxies)
 
     proxies.each do |region, list|
       spawn do
+        info = HTTP::Params.new({
+          "reason" => [info["reason"]],
+        })
+
         list.each do |proxy|
           begin
             client = HTTPClient.new(YT_URL)
@@ -563,6 +567,11 @@ def fetch_video(id, proxies)
           rescue ex
           end
         end
+
+        # If none of the proxies we tried returned a valid response
+        if info["reason"]?
+          bypass_channel.send(nil)
+        end
       end
     end