better spoof requests

src/invidious/yt_backend/connection_pool.cr

@@ -13,9 +13,6 @@ def add_yt_headers(request)
   request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
   request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
   request.headers["Accept-Language"] ||= "en-us,en;q=0.5"
-  return if request.resource.starts_with? "/sorry/index"
-  request.headers["x-youtube-client-name"] ||= "1"
-  request.headers["x-youtube-client-version"] ||= "2.20200609"
   # Preserve original cookies and add new YT consent cookie for EU servers
   request.headers["Cookie"] = "#{request.headers["cookie"]?}; CONSENT=YES+"
   if !CONFIG.cookies.empty?

src/invidious/yt_backend/youtube_api.cr

@@ -8,11 +8,16 @@ module YoutubeAPI
   private DEFAULT_API_KEY = "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8"
 
   private ANDROID_APP_VERSION = "17.33.42"
-  private ANDROID_USER_AGENT  = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US)"
+  # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1308
+  private ANDROID_USER_AGENT  = "com.google.android.youtube/17.33.42 (Linux; U; Android 12; US) gzip"
   private ANDROID_SDK_VERSION = 31_i64
   private ANDROID_VERSION     = "12"
   private IOS_APP_VERSION     = "17.33.2"
-  private WINDOWS_VERSION     = "10.0"
+  # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1330
+  private IOS_USER_AGENT = "com.google.ios.youtube/17.33.2 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)"
+  # github.com/TeamNewPipe/NewPipeExtractor/blob/943b7c033bb9d07ead63ddab4441c287653e4384/extractor/src/main/java/org/schabi/newpipe/extractor/services/youtube/YoutubeParsingHelper.java#L1224
+  private IOS_VERSION     = "15.6.0.19G71"
+  private WINDOWS_VERSION = "10.0"
 
   # Enumerate used to select one of the clients supported by the API
   enum ClientType
@@ -37,6 +42,7 @@ module YoutubeAPI
   HARDCODED_CLIENTS = {
     ClientType::Web => {
       name:       "WEB",
+      name_proto: "1",
       version:    "2.20220804.07.00",
       api_key:    DEFAULT_API_KEY,
       screen:     "WATCH_FULL_SCREEN",
@@ -45,7 +51,8 @@ module YoutubeAPI
       platform:   "DESKTOP",
     },
     ClientType::WebEmbeddedPlayer => {
-      name:       "WEB_EMBEDDED_PLAYER", # 56
+      name:       "WEB_EMBEDDED_PLAYER",
+      name_proto: "56",
       version:    "1.20220803.01.00",
       api_key:    DEFAULT_API_KEY,
       screen:     "EMBED",
@@ -55,6 +62,7 @@ module YoutubeAPI
     },
     ClientType::WebMobile => {
       name:       "MWEB",
+      name_proto: "2",
       version:    "2.20220805.01.00",
       api_key:    DEFAULT_API_KEY,
       os_name:    "Android",
@@ -63,6 +71,7 @@ module YoutubeAPI
     },
     ClientType::WebScreenEmbed => {
       name:       "WEB",
+      name_proto: "1",
       version:    "2.20220804.00.00",
       api_key:    DEFAULT_API_KEY,
       screen:     "EMBED",
@@ -75,6 +84,7 @@ module YoutubeAPI
 
     ClientType::Android => {
       name:                "ANDROID",
+      name_proto:          "3",
       version:             ANDROID_APP_VERSION,
       api_key:             "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
       android_sdk_version: ANDROID_SDK_VERSION,
@@ -84,12 +94,14 @@ module YoutubeAPI
       platform:            "MOBILE",
     },
     ClientType::AndroidEmbeddedPlayer => {
-      name:    "ANDROID_EMBEDDED_PLAYER", # 55
+      name:       "ANDROID_EMBEDDED_PLAYER",
-      version: ANDROID_APP_VERSION,
+      name_proto: "55",
-      api_key: DEFAULT_API_KEY,
+      version:    ANDROID_APP_VERSION,
+      api_key:    DEFAULT_API_KEY,
     },
     ClientType::AndroidScreenEmbed => {
-      name:                "ANDROID", # 3
+      name:                "ANDROID",
+      name_proto:          "3",
       version:             ANDROID_APP_VERSION,
       api_key:             DEFAULT_API_KEY,
       screen:              "EMBED",
@@ -103,33 +115,56 @@ module YoutubeAPI
     # IOS
 
     ClientType::IOS => {
-      name:    "IOS", # 5
+      name:         "IOS",
-      version: IOS_APP_VERSION,
+      name_proto:   "5",
-      api_key: "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc",
+      version:      IOS_APP_VERSION,
+      api_key:      "AIzaSyB-63vPrdThhKuerbB2N_l7Kwwcxj6yUAc",
+      user_agent:   IOS_USER_AGENT,
+      device_make:  "Apple",
+      device_model: "iPhone14,5",
+      os_name:      "iPhone",
+      os_version:   IOS_VERSION,
+      platform:     "MOBILE",
     },
     ClientType::IOSEmbedded => {
-      name:    "IOS_MESSAGES_EXTENSION", # 66
+      name:         "IOS_MESSAGES_EXTENSION",
-      version: IOS_APP_VERSION,
+      name_proto:   "66",
-      api_key: DEFAULT_API_KEY,
+      version:      IOS_APP_VERSION,
+      api_key:      DEFAULT_API_KEY,
+      user_agent:   IOS_USER_AGENT,
+      device_make:  "Apple",
+      device_model: "iPhone14,5",
+      os_name:      "iPhone",
+      os_version:   IOS_VERSION,
+      platform:     "MOBILE",
     },
     ClientType::IOSMusic => {
-      name:    "IOS_MUSIC", # 26
+      name:         "IOS_MUSIC",
-      version: "4.32",
+      name_proto:   "26",
-      api_key: "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s",
+      version:      "5.21",
+      api_key:      "AIzaSyBAETezhkwP0ZWA02RsqT1zu78Fpt0bC_s",
+      user_agent:   "com.google.ios.youtubemusic/5.21 (iPhone14,5; U; CPU iOS 15_6 like Mac OS X;)",
+      device_make:  "Apple",
+      device_model: "iPhone14,5",
+      os_name:      "iPhone",
+      os_version:   IOS_VERSION,
+      platform:     "MOBILE",
     },
 
     # TV app
 
     ClientType::TvHtml5 => {
-      name:    "TVHTML5", # 7
+      name:       "TVHTML5",
-      version: "7.20220325",
+      name_proto: "7",
-      api_key: DEFAULT_API_KEY,
+      version:    "7.20220325",
+      api_key:    DEFAULT_API_KEY,
     },
     ClientType::TvHtml5ScreenEmbed => {
-      name:    "TVHTML5_SIMPLY_EMBEDDED_PLAYER", # 85
+      name:       "TVHTML5_SIMPLY_EMBEDDED_PLAYER",
-      version: "2.0",
+      name_proto: "85",
-      api_key: DEFAULT_API_KEY,
+      version:    "2.0",
-      screen:  "EMBED",
+      api_key:    DEFAULT_API_KEY,
+      screen:     "EMBED",
     },
   }
 
@@ -183,6 +218,10 @@ module YoutubeAPI
       HARDCODED_CLIENTS[@client_type][:name]
     end
 
+    def name_proto : String
+      HARDCODED_CLIENTS[@client_type][:name_proto]
+    end
+
     # :ditto:
     def version : String
       HARDCODED_CLIENTS[@client_type][:version]
@@ -210,6 +249,14 @@ module YoutubeAPI
       HARDCODED_CLIENTS[@client_type][:os_name]?
     end
 
+    def device_make : String?
+      HARDCODED_CLIENTS[@client_type][:device_make]?
+    end
+
+    def device_model : String?
+      HARDCODED_CLIENTS[@client_type][:device_model]?
+    end
+
     def os_version : String?
       HARDCODED_CLIENTS[@client_type][:os_version]?
     end
@@ -265,6 +312,14 @@ module YoutubeAPI
       client_context["client"]["androidSdkVersion"] = android_sdk_version
     end
 
+    if device_make = client_config.device_make
+      client_context["client"]["deviceMake"] = device_make
+    end
+
+    if device_model = client_config.device_model
+      client_context["client"]["deviceModel"] = device_model
+    end
+
     if os_name = client_config.os_name
       client_context["client"]["osName"] = os_name
     end
@@ -524,6 +579,8 @@ module YoutubeAPI
       "Content-Type"              => "application/json; charset=UTF-8",
       "Accept-Encoding"           => "gzip, deflate",
       "x-goog-api-format-version" => "2",
+      "x-youtube-client-name"     => client_config.name_proto,
+      "x-youtube-client-version"  => client_config.version,
     }
 
     if user_agent = client_config.user_agent