From eb8fcc9e88bf71694e7d9e40600ef3fd334bc46e Mon Sep 17 00:00:00 2001 From: syeopite Date: Fri, 25 Aug 2023 16:08:02 -0700 Subject: [PATCH 1/4] Add support for using HTTP proxies --- config/config.example.yml | 11 +++++++++++ shard.lock | 10 +++++++--- shard.yml | 3 +++ src/invidious.cr | 1 + src/invidious/config.cr | 11 +++++++++++ src/invidious/yt_backend/connection_pool.cr | 18 ++++++++++++++++++ 6 files changed, 51 insertions(+), 3 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 38085a20..9fdc4eda 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -160,6 +160,17 @@ https_only: false ## #force_resolve: +## +## Configuration for using a HTTP proxy +## +## If unset, then no HTTP proxy will be used. +## +http_proxy: + user: + password: + host: + port: + ## ## Use Innertube's transcripts API instead of timedtext for closed captions diff --git a/shard.lock b/shard.lock index efb60a59..85928c47 100644 --- a/shard.lock +++ b/shard.lock @@ -6,11 +6,11 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.1 + version: 0.1.3 backtracer: git: https://github.com/sija/backtracer.cr.git - version: 1.2.1 + version: 1.2.2 db: git: https://github.com/crystal-lang/crystal-db.git @@ -20,6 +20,10 @@ shards: git: https://github.com/crystal-loot/exception_page.git version: 0.2.2 + http_proxy: + git: https://github.com/mamantoha/http_proxy.git + version: 0.10.1 + kemal: git: https://github.com/kemalcr/kemal.git version: 1.1.2 @@ -42,7 +46,7 @@ shards: spectator: git: https://github.com/icy-arctic-fox/spectator.git - version: 0.10.4 + version: 0.10.6 sqlite3: git: https://github.com/crystal-lang/crystal-sqlite3.git diff --git a/shard.yml b/shard.yml index be06a7df..ddd510d4 100644 --- a/shard.yml +++ b/shard.yml @@ -28,6 +28,9 @@ dependencies: athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 + http_proxy: + github: mamantoha/http_proxy + version: ~> 0.10.1 development_dependencies: spectator: diff --git a/src/invidious.cr b/src/invidious.cr index e0bd0101..d4114386 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -23,6 +23,7 @@ require "kilt" require "./ext/kemal_content_for.cr" require "./ext/kemal_static_file_handler.cr" +require "http_proxy" require "athena-negotiation" require "openssl/hmac" require "option_parser" diff --git a/src/invidious/config.cr b/src/invidious/config.cr index 09c2168b..e12054d0 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -54,6 +54,15 @@ struct ConfigPreferences end end +struct HTTPProxyConfig + include YAML::Serializable + + property user : String + property password : String + property host : String + property port : Int32 +end + class Config include YAML::Serializable @@ -126,6 +135,8 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 + # HTTP Proxy configuration + property http_proxy : HTTPProxyConfig? = nil # Use Innertube's transcripts API instead of timedtext for closed captions property use_innertube_for_captions : Bool = false diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index d3dbcc0e..0e4d8aff 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,12 +26,14 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy begin response = yield conn rescue ex conn.close conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -46,6 +48,7 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) + conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -66,6 +69,8 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds + client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + return client end @@ -77,3 +82,16 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false, &block) client.close end end + +def make_configured_http_proxy_client + # This method is only called when configuration for an HTTP proxy are set + config_proxy = CONFIG.http_proxy.not_nil! + + return HTTP::Proxy::Client.new( + config_proxy.host, + config_proxy.port, + + username: config_proxy.user, + password: config_proxy.password, + ) +end From 3b471ae964ad0122c81205965c221a352e3a658f Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 4 Oct 2023 14:36:04 -0400 Subject: [PATCH 2/4] Automatically initialize proxy via stdlib override --- .../helpers/crystal_class_overrides.cr | 34 +++++++++++++++++++ src/invidious/yt_backend/connection_pool.cr | 7 ++-- 2 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index bf56d826..71038703 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -18,6 +18,40 @@ end class HTTP::Client property family : Socket::Family = Socket::Family::UNSPEC + # Override stdlib to automatically initialize proxy if configured + # + # Accurate as of crystal 1.10.1 + + def initialize(@host : String, port = nil, tls : TLSContext = nil) + check_host_only(@host) + + {% if flag?(:without_openssl) %} + if tls + raise "HTTP::Client TLS is disabled because `-D without_openssl` was passed at compile time" + end + @tls = nil + {% else %} + @tls = case tls + when true + OpenSSL::SSL::Context::Client.new + when OpenSSL::SSL::Context::Client + tls + when false, nil + nil + end + {% end %} + + @port = (port || (@tls ? 443 : 80)).to_i + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + + def initialize(@io : IO, @host = "", @port = 80) + @reconnect = false + + self.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + end + private def io io = @io return io if io diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 0e4d8aff..f34f48c5 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -26,13 +26,15 @@ struct YoutubeConnectionPool def client(&block) conn = pool.checkout + # Proxy needs to be reinstated every time we get a client from the pool conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy + begin response = yield conn rescue ex conn.close - conn = HTTP::Client.new(url) + conn = HTTP::Client.new(url) conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC @@ -48,7 +50,6 @@ struct YoutubeConnectionPool private def build_pool DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do conn = HTTP::Client.new(url) - conn.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy conn.family = CONFIG.force_resolve conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -69,8 +70,6 @@ def make_client(url : URI, region = nil, force_resolve : Bool = false) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds - client.proxy = make_configured_http_proxy_client() if CONFIG.http_proxy - return client end From ccb2a6c58ef9b5b7e1c47938cfa4ec574a8560c7 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sun, 28 Apr 2024 21:34:05 -0700 Subject: [PATCH 3/4] Bump http_proxy to v0.10.3 --- shard.lock | 4 ++-- shard.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shard.lock b/shard.lock index 85928c47..99b4e99c 100644 --- a/shard.lock +++ b/shard.lock @@ -6,7 +6,7 @@ shards: athena-negotiation: git: https://github.com/athena-framework/negotiation.git - version: 0.1.3 + version: 0.1.1 backtracer: git: https://github.com/sija/backtracer.cr.git @@ -22,7 +22,7 @@ shards: http_proxy: git: https://github.com/mamantoha/http_proxy.git - version: 0.10.1 + version: 0.10.3 kemal: git: https://github.com/kemalcr/kemal.git diff --git a/shard.yml b/shard.yml index ddd510d4..70ebad0a 100644 --- a/shard.yml +++ b/shard.yml @@ -30,7 +30,7 @@ dependencies: version: ~> 0.1.1 http_proxy: github: mamantoha/http_proxy - version: ~> 0.10.1 + version: ~> 0.10.3 development_dependencies: spectator: From 6b7e7301009e1a9fc2b536bd8d8de04fb8e22ec0 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 22 May 2024 13:10:46 -0700 Subject: [PATCH 4/4] Validate override for crystal 1.12.1 --- src/invidious/helpers/crystal_class_overrides.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/helpers/crystal_class_overrides.cr b/src/invidious/helpers/crystal_class_overrides.cr index 71038703..a7d2a5e6 100644 --- a/src/invidious/helpers/crystal_class_overrides.cr +++ b/src/invidious/helpers/crystal_class_overrides.cr @@ -20,7 +20,7 @@ class HTTP::Client # Override stdlib to automatically initialize proxy if configured # - # Accurate as of crystal 1.10.1 + # Accurate as of crystal 1.12.1 def initialize(@host : String, port = nil, tls : TLSContext = nil) check_host_only(@host)