From 065c104f274b88495dc331f32dd3dc1cf2c99646 Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 24 May 2021 06:45:50 -0700 Subject: [PATCH 1/5] Upgrade to crystal 1.0.0 --- shard.lock | 18 +++++++----------- shard.yml | 11 ++++------- src/invidious.cr | 2 +- src/invidious/helpers/utils.cr | 8 ++++---- src/invidious/jobs/bypass_captcha_job.cr | 2 +- src/invidious/routes/login.cr | 4 ++-- src/invidious/routes/preferences.cr | 8 ++++---- src/invidious/users.cr | 2 +- 8 files changed, 24 insertions(+), 31 deletions(-) diff --git a/shard.lock b/shard.lock index 5ed730b8..9a087f7a 100644 --- a/shard.lock +++ b/shard.lock @@ -2,19 +2,19 @@ version: 2.0 shards: db: git: https://github.com/crystal-lang/crystal-db.git - version: 0.10.0 + version: 0.10.1 exception_page: git: https://github.com/crystal-loot/exception_page.git - version: 0.1.4 + version: 0.1.5 kemal: git: https://github.com/kemalcr/kemal.git - version: 0.27.0 + version: 1.0.0 kilt: git: https://github.com/jeromegn/kilt.git - version: 0.4.0 + version: 0.4.1 lsquic: git: https://github.com/iv-org/lsquic.cr.git @@ -22,19 +22,15 @@ shards: pg: git: https://github.com/will/crystal-pg.git - version: 0.23.1 - - pool: - git: https://github.com/ysbaddaden/pool.git - version: 0.2.3 + version: 0.23.2 protodec: git: https://github.com/iv-org/protodec.git - version: 0.1.3 + version: 0.1.4 radix: git: https://github.com/luislavena/radix.git - version: 0.3.9 + version: 0.4.1 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/shard.yml b/shard.yml index a15b3ea5..f7746cb5 100644 --- a/shard.yml +++ b/shard.yml @@ -12,23 +12,20 @@ targets: dependencies: pg: github: will/crystal-pg - version: ~> 0.23.1 + version: ~> 0.23.2 sqlite3: github: crystal-lang/crystal-sqlite3 version: ~> 0.18.0 kemal: github: kemalcr/kemal - version: ~> 0.27.0 - pool: - github: ysbaddaden/pool - version: ~> 0.2.3 + version: ~> 1.0.0 protodec: github: iv-org/protodec - version: ~> 0.1.3 + version: ~> 0.1.4 lsquic: github: iv-org/lsquic.cr version: ~> 2.18.1-1 -crystal: 0.36.1 +crystal: 1.0.0 license: AGPLv3 diff --git a/src/invidious.cr b/src/invidious.cr index ae20e13e..de89b9c3 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -421,7 +421,7 @@ get "/modify_notifications" do |env| html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) html.cookies.each do |cookie| if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name if cookies[cookie.name]? diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index 10d4e6b6..20107fad 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -1,5 +1,5 @@ require "lsquic" -require "pool/connection" +require "db" def add_yt_headers(request) request.headers["user-agent"] ||= "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36" @@ -20,7 +20,7 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : ConnectionPool(QUIC::Client | HTTP::Client) + property pool : DB::Pool(QUIC::Client | HTTP::Client) def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) @url = url @@ -43,7 +43,7 @@ struct YoutubeConnectionPool conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" response = yield conn ensure - pool.checkin(conn) + pool.release(conn) end end @@ -51,7 +51,7 @@ struct YoutubeConnectionPool end private def build_pool(use_quic) - ConnectionPool(QUIC::Client | HTTP::Client).new(capacity: capacity, timeout: timeout) do + DB::Pool(QUIC::Client | HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do if use_quic conn = QUIC::Client.new(url) else diff --git a/src/invidious/jobs/bypass_captcha_job.cr b/src/invidious/jobs/bypass_captcha_job.cr index 4269e123..e68b81e6 100644 --- a/src/invidious/jobs/bypass_captcha_job.cr +++ b/src/invidious/jobs/bypass_captcha_job.cr @@ -112,7 +112,7 @@ class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob headers = HTTP::Headers{ "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0], } - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) cookies.each { |cookie| CONFIG.cookies << cookie } diff --git a/src/invidious/routes/login.cr b/src/invidious/routes/login.cr index ffe5f568..f9e6ea6c 100644 --- a/src/invidious/routes/login.cr +++ b/src/invidious/routes/login.cr @@ -238,7 +238,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute traceback << "Logging in..." location = URI.parse(challenge_results[0][-1][2].to_s) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) headers.delete("Content-Type") headers.delete("Google-Accounts-XSRF") @@ -261,7 +261,7 @@ class Invidious::Routes::Login < Invidious::Routes::BaseRoute location = login.headers["Location"]?.try { |u| URI.parse(u) } end - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) sid = cookies["SID"]?.try &.value if !sid raise "Couldn't get SID." diff --git a/src/invidious/routes/preferences.cr b/src/invidious/routes/preferences.cr index cfdad443..f98c7a5e 100644 --- a/src/invidious/routes/preferences.cr +++ b/src/invidious/routes/preferences.cr @@ -198,10 +198,10 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute end if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) end end @@ -250,10 +250,10 @@ class Invidious::Routes::PreferencesRoute < Invidious::Routes::BaseRoute end if CONFIG.domain - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", domain: "#{CONFIG.domain}", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) else - env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: preferences, expires: Time.utc + 2.years, + env.response.cookies["PREFS"] = HTTP::Cookie.new(name: "PREFS", value: URI.encode_www_form(preferences), expires: Time.utc + 2.years, secure: secure, http_only: true) end end diff --git a/src/invidious/users.cr b/src/invidious/users.cr index 5dfd80bb..d774ee12 100644 --- a/src/invidious/users.cr +++ b/src/invidious/users.cr @@ -462,7 +462,7 @@ def subscribe_ajax(channel_id, action, env_headers) html = YT_POOL.client &.get("/subscription_manager?disable_polymer=1", headers) - cookies = HTTP::Cookies.from_headers(headers) + cookies = HTTP::Cookies.from_client_headers(headers) html.cookies.each do |cookie| if {"VISITOR_INFO1_LIVE", "YSC", "SIDCC"}.includes? cookie.name if cookies[cookie.name]? From 0a87ba6930948ebfec9a3ad2e336c0c7e6037d8a Mon Sep 17 00:00:00 2001 From: syeopite Date: Mon, 24 May 2021 06:52:55 -0700 Subject: [PATCH 2/5] Decode URL encoded preference cookie when parsing --- src/invidious.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious.cr b/src/invidious.cr index de89b9c3..ebba52b1 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -166,7 +166,7 @@ end before_all do |env| preferences = begin - Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}") + Preferences.from_json(URI.decode_www_form(env.request.cookies["PREFS"]?.try &.value || "{}")) rescue Preferences.from_json("{}") end From 392a363d02142026f717f44381b043f199918489 Mon Sep 17 00:00:00 2001 From: syeopite Date: Tue, 8 Jun 2021 13:20:46 -0700 Subject: [PATCH 3/5] Update Dockerfile for crystal 1.0 --- docker/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/Dockerfile b/docker/Dockerfile index 87884403..b8e5af8a 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM crystallang/crystal:0.36.1-alpine AS builder +FROM crystallang/crystal:1.0.0-alpine AS builder RUN apk add --no-cache curl sqlite-static yaml-static WORKDIR /invidious COPY ./shard.yml ./shard.yml From 114cac99e781aefb54b5d377fcebb2d6c6274417 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 9 Jun 2021 16:34:54 -0700 Subject: [PATCH 4/5] Update CI's crystal version --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 66aacff9..54e70d0a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: - name: Install Crystal uses: oprypin/install-crystal@v1.2.4 with: - crystal: 0.36.1 + crystal: 1.0.0 - name: Cache Shards uses: actions/cache@v2 From 8701de64cebc64860ebcf00262ac0d44d01b0a90 Mon Sep 17 00:00:00 2001 From: syeopite Date: Thu, 10 Jun 2021 04:49:19 -0700 Subject: [PATCH 5/5] Bump lsquic.cr shard version to v2.18.1-2 --- shard.lock | 2 +- shard.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shard.lock b/shard.lock index 9a087f7a..35d1aefd 100644 --- a/shard.lock +++ b/shard.lock @@ -18,7 +18,7 @@ shards: lsquic: git: https://github.com/iv-org/lsquic.cr.git - version: 2.18.1-1 + version: 2.18.1-2 pg: git: https://github.com/will/crystal-pg.git diff --git a/shard.yml b/shard.yml index f7746cb5..fca1ce02 100644 --- a/shard.yml +++ b/shard.yml @@ -24,7 +24,7 @@ dependencies: version: ~> 0.1.4 lsquic: github: iv-org/lsquic.cr - version: ~> 2.18.1-1 + version: ~> 2.18.1-2 crystal: 1.0.0