From c5fe96e93603db58d6767928eedc658e8b58e59f Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 26 Jul 2023 07:19:12 -0700 Subject: [PATCH 1/2] Remove lsquic from codebase --- config/config.example.yml | 21 --- shard.lock | 4 - shard.yml | 3 - src/invidious.cr | 2 +- src/invidious/config.cr | 2 - src/invidious/routes/images.cr | 142 +++----------------- src/invidious/yt_backend/connection_pool.cr | 37 +---- src/invidious/yt_backend/youtube_api.cr | 14 +- 8 files changed, 32 insertions(+), 193 deletions(-) diff --git a/config/config.example.yml b/config/config.example.yml index 34070fe5..e925a5e3 100644 --- a/config/config.example.yml +++ b/config/config.example.yml @@ -140,27 +140,6 @@ https_only: false ## #pool_size: 100 -## -## Enable/Disable the use of QUIC (HTTP/3) when connecting -## to the youtube API and websites ('youtube.com', 'ytimg.com'). -## QUIC's main advantages are its lower latency and lower bandwidth -## use, compared to its predecessors. However, the current version -## of QUIC used in invidious is still based on the IETF draft 31, -## meaning that the underlying library may still not be fully -## optimized. You can read more about QUIC at the link below: -## https://datatracker.ietf.org/doc/html/draft-ietf-quic-transport-31 -## -## Note: you should try both options and see what is the best for your -## instance. In general QUIC is recommended for public instances. Your -## mileage may vary. -## -## Note 2: Using QUIC prevents some captcha challenges from appearing. -## See: https://github.com/iv-org/invidious/issues/957#issuecomment-576424042 -## -## Accepted values: true, false -## Default: false -## -#use_quic: false ## ## Additional cookies to be sent when requesting the youtube API. diff --git a/shard.lock b/shard.lock index 235e4c25..55fcfe46 100644 --- a/shard.lock +++ b/shard.lock @@ -24,10 +24,6 @@ shards: git: https://github.com/jeromegn/kilt.git version: 0.6.1 - lsquic: - git: https://github.com/iv-org/lsquic.cr.git - version: 2.18.1-2 - pg: git: https://github.com/will/crystal-pg.git version: 0.24.0 diff --git a/shard.yml b/shard.yml index 7ee0bb2a..e929160d 100644 --- a/shard.yml +++ b/shard.yml @@ -25,9 +25,6 @@ dependencies: protodec: github: iv-org/protodec version: ~> 0.1.5 - lsquic: - github: iv-org/lsquic.cr - version: ~> 2.18.1-2 athena-negotiation: github: athena-framework/negotiation version: ~> 0.1.1 diff --git a/src/invidious.cr b/src/invidious.cr index 84e1895d..e0bd0101 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -90,7 +90,7 @@ SOFTWARE = { "branch" => "#{CURRENT_BRANCH}", } -YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size, use_quic: CONFIG.use_quic) +YT_POOL = YoutubeConnectionPool.new(YT_URL, capacity: CONFIG.pool_size) # CLI Kemal.config.extra_options do |parser| diff --git a/src/invidious/config.cr b/src/invidious/config.cr index e5f1e822..cee33ce1 100644 --- a/src/invidious/config.cr +++ b/src/invidious/config.cr @@ -126,8 +126,6 @@ class Config property host_binding : String = "0.0.0.0" # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`) property pool_size : Int32 = 100 - # Use quic transport for youtube api - property use_quic : Bool = false # Saved cookies in "name1=value1; name2=value2..." format @[YAML::Field(converter: Preferences::StringToCookies)] diff --git a/src/invidious/routes/images.cr b/src/invidious/routes/images.cr index 594a7869..b6a2e110 100644 --- a/src/invidious/routes/images.cr +++ b/src/invidious/routes/images.cr @@ -3,17 +3,7 @@ module Invidious::Routes::Images def self.ggpht(env) url = env.request.path.lchop("/ggpht") - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "yt3.ggpht.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -42,22 +32,9 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + HTTP::Client.get("https://yt3.ggpht.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end @@ -78,10 +55,6 @@ module Invidious::Routes::Images headers = HTTP::Headers.new - {% unless flag?(:disable_quic) %} - headers[":authority"] = "#{authority}.ytimg.com" - {% end %} - REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? headers[header] = env.request.headers[header] @@ -107,22 +80,9 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + HTTP::Client.get("https://#{authority}.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end @@ -133,17 +93,7 @@ module Invidious::Routes::Images name = env.params.url["name"] url = env.request.resource - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i9.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new REQUEST_HEADERS_WHITELIST.each do |header| if env.request.headers[header]? @@ -169,22 +119,9 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + HTTP::Client.get("https://i9.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end @@ -223,41 +160,16 @@ module Invidious::Routes::Images id = env.params.url["id"] name = env.params.url["name"] - headers = ( - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - HTTP::Headers{":authority" => "i.ytimg.com"} - else - HTTP::Headers.new - end - {% else %} - HTTP::Headers.new - {% end %} - ) + headers = HTTP::Headers.new if name == "maxres.jpg" build_thumbnails(id).each do |thumb| thumbnail_resource_path = "/vi/#{id}/#{thumb[:url]}.jpg" - # Logic here is short enough that manually typing them out should be fine. - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - if YT_POOL.client &.head(thumbnail_resource_path, headers).status_code == 200 - name = thumb[:url] + ".jpg" - break - end - else - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 - name = thumb[:url] + ".jpg" - break - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 - name = thumb[:url] + ".jpg" - break - end - {% end %} + # This can likely be optimized into a (small) pool sometime in the future. + if HTTP::Client.head("https://i.ytimg.com#{thumbnail_resource_path}").status_code == 200 + name = thumb[:url] + ".jpg" + break + end end end @@ -287,22 +199,10 @@ module Invidious::Routes::Images } begin - {% unless flag?(:disable_quic) %} - if CONFIG.use_quic - YT_POOL.client &.get(url, headers) do |resp| - return request_proc.call(resp) - end - else - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - end - {% else %} - # This can likely be optimized into a (small) pool sometime in the future. - HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| - return request_proc.call(resp) - end - {% end %} + # This can likely be optimized into a (small) pool sometime in the future. + HTTP::Client.get("https://i.ytimg.com#{url}") do |resp| + return request_proc.call(resp) + end rescue ex end end diff --git a/src/invidious/yt_backend/connection_pool.cr b/src/invidious/yt_backend/connection_pool.cr index 658731cf..e9eb726c 100644 --- a/src/invidious/yt_backend/connection_pool.cr +++ b/src/invidious/yt_backend/connection_pool.cr @@ -1,11 +1,3 @@ -{% unless flag?(:disable_quic) %} - require "lsquic" - - alias HTTPClientType = QUIC::Client | HTTP::Client -{% else %} - alias HTTPClientType = HTTP::Client -{% end %} - def add_yt_headers(request) if request.headers["User-Agent"] == "Crystal" request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36" @@ -26,11 +18,11 @@ struct YoutubeConnectionPool property! url : URI property! capacity : Int32 property! timeout : Float64 - property pool : DB::Pool(HTTPClientType) + property pool : DB::Pool(HTTP::Client) - def initialize(url : URI, @capacity = 5, @timeout = 5.0, use_quic = true) + def initialize(url : URI, @capacity = 5, @timeout = 5.0) @url = url - @pool = build_pool(use_quic) + @pool = build_pool() end def client(region = nil, &block) @@ -43,11 +35,7 @@ struct YoutubeConnectionPool response = yield conn rescue ex conn.close - {% unless flag?(:disable_quic) %} - conn = CONFIG.use_quic ? QUIC::Client.new(url) : HTTP::Client.new(url) - {% else %} - conn = HTTP::Client.new(url) - {% end %} + conn = HTTP::Client.new(url) conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC @@ -61,19 +49,9 @@ struct YoutubeConnectionPool response end - private def build_pool(use_quic) - DB::Pool(HTTPClientType).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do - conn = nil # Declare - {% unless flag?(:disable_quic) %} - if use_quic - conn = QUIC::Client.new(url) - else - conn = HTTP::Client.new(url) - end - {% else %} - conn = HTTP::Client.new(url) - {% end %} - + private def build_pool + DB::Pool(HTTP::Client).new(initial_pool_size: 0, max_pool_size: capacity, max_idle_pool_size: capacity, checkout_timeout: timeout) do + conn = HTTP::Client.new(url) conn.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::INET conn.family = Socket::Family::INET if conn.family == Socket::Family::UNSPEC conn.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" @@ -83,7 +61,6 @@ struct YoutubeConnectionPool end def make_client(url : URI, region = nil) - # TODO: Migrate any applicable endpoints to QUIC client = HTTPClient.new(url, OpenSSL::SSL::Context::Client.insecure) client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com" diff --git a/src/invidious/yt_backend/youtube_api.cr b/src/invidious/yt_backend/youtube_api.cr index 3dd9e9d8..aef9ddd9 100644 --- a/src/invidious/yt_backend/youtube_api.cr +++ b/src/invidious/yt_backend/youtube_api.cr @@ -595,17 +595,9 @@ module YoutubeAPI LOGGER.trace("YoutubeAPI: POST data: #{data}") # Send the POST request - if {{ !flag?(:disable_quic) }} && CONFIG.use_quic - # Using QUIC client - body = YT_POOL.client(client_config.proxy_region, - &.post(url, headers: headers, body: data.to_json) - ).body - else - # Using HTTP client - body = YT_POOL.client(client_config.proxy_region) do |client| - client.post(url, headers: headers, body: data.to_json) do |response| - self._decompress(response.body_io, response.headers["Content-Encoding"]?) - end + body = YT_POOL.client(client_config.proxy_region) do |client| + client.post(url, headers: headers, body: data.to_json) do |response| + self._decompress(response.body_io, response.headers["Content-Encoding"]?) end end From a8ba02051b261a634050ea7f621451d84ca61607 Mon Sep 17 00:00:00 2001 From: syeopite Date: Wed, 26 Jul 2023 07:25:19 -0700 Subject: [PATCH 2/2] Remove(?) lsquic from make and docker files --- .github/workflows/container-release.yml | 29 ++----------------------- Makefile | 6 ----- docker/Dockerfile | 11 +--------- docker/Dockerfile.arm64 | 11 +--------- 4 files changed, 4 insertions(+), 53 deletions(-) diff --git a/.github/workflows/container-release.yml b/.github/workflows/container-release.yml index 86aec94f..13bbf34c 100644 --- a/.github/workflows/container-release.yml +++ b/.github/workflows/container-release.yml @@ -52,7 +52,7 @@ jobs: username: ${{ secrets.QUAY_USERNAME }} password: ${{ secrets.QUAY_PASSWORD }} - - name: Build and push Docker AMD64 image without QUIC for Push Event + - name: Build and push Docker AMD64 image for Push Event if: github.ref == 'refs/heads/master' uses: docker/build-push-action@v3 with: @@ -64,9 +64,8 @@ jobs: tags: quay.io/invidious/invidious:${{ github.sha }},quay.io/invidious/invidious:latest build-args: | "release=1" - "disable_quic=1" - - name: Build and push Docker ARM64 image without QUIC for Push Event + - name: Build and push Docker ARM64 image for Push Event if: github.ref == 'refs/heads/master' uses: docker/build-push-action@v3 with: @@ -78,28 +77,4 @@ jobs: tags: quay.io/invidious/invidious:${{ github.sha }}-arm64,quay.io/invidious/invidious:latest-arm64 build-args: | "release=1" - "disable_quic=1" - - name: Build and push Docker AMD64 image with QUIC for Push Event - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 - with: - context: . - file: docker/Dockerfile - platforms: linux/amd64 - labels: quay.expires-after=12w - push: true - tags: quay.io/invidious/invidious:${{ github.sha }}-quic,quay.io/invidious/invidious:latest-quic - build-args: release=1 - - - name: Build and push Docker ARM64 image with QUIC for Push Event - if: github.ref == 'refs/heads/master' - uses: docker/build-push-action@v3 - with: - context: . - file: docker/Dockerfile.arm64 - platforms: linux/arm64/v8 - labels: quay.expires-after=12w - push: true - tags: quay.io/invidious/invidious:${{ github.sha }}-arm64-quic,quay.io/invidious/invidious:latest-arm64-quic - build-args: release=1 diff --git a/Makefile b/Makefile index d4657792..9eb195df 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,6 @@ RELEASE := 1 STATIC := 0 -DISABLE_QUIC := 1 NO_DBG_SYMBOLS := 0 @@ -27,10 +26,6 @@ else FLAGS += --debug endif -ifeq ($(DISABLE_QUIC), 1) - FLAGS += -Ddisable_quic -endif - ifeq ($(API_ONLY), 1) FLAGS += -Dapi_only endif @@ -115,7 +110,6 @@ help: @echo " STATIC Link libraries statically (Default: 0)" @echo "" @echo " API_ONLY Build invidious without a GUI (Default: 0)" - @echo " DISABLE_QUIC Disable support for QUIC (Default: 0)" @echo " NO_DBG_SYMBOLS Strip debug symbols (Default: 0)" diff --git a/docker/Dockerfile b/docker/Dockerfile index 57864883..761bbdca 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -2,15 +2,12 @@ FROM crystallang/crystal:1.4.1-alpine AS builder RUN apk add --no-cache sqlite-static yaml-static ARG release -ARG disable_quic WORKDIR /invidious COPY ./shard.yml ./shard.yml COPY ./shard.lock ./shard.lock RUN shards install --production -COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a - COPY ./src/ ./src/ # TODO: .git folder is required for building – this is destructive. # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. @@ -24,13 +21,7 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml RUN crystal spec --warnings all \ --link-flags "-lxml2 -llzma" -RUN if [[ "${release}" == 1 && "${disable_quic}" == 1 ]] ; then \ - crystal build ./src/invidious.cr \ - --release \ - -Ddisable_quic \ - --static --warnings all \ - --link-flags "-lxml2 -llzma"; \ - elif [[ "${release}" == 1 ]] ; then \ +RUN if [[ "${release}" == 1 ]] ; then \ crystal build ./src/invidious.cr \ --release \ --static --warnings all \ diff --git a/docker/Dockerfile.arm64 b/docker/Dockerfile.arm64 index 10135efb..cf9231fb 100644 --- a/docker/Dockerfile.arm64 +++ b/docker/Dockerfile.arm64 @@ -2,15 +2,12 @@ FROM alpine:3.16 AS builder RUN apk add --no-cache 'crystal=1.4.1-r0' shards sqlite-static yaml-static yaml-dev libxml2-dev zlib-static openssl-libs-static openssl-dev musl-dev ARG release -ARG disable_quic WORKDIR /invidious COPY ./shard.yml ./shard.yml COPY ./shard.lock ./shard.lock RUN shards install --production -COPY --from=quay.io/invidious/lsquic-compiled /root/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a - COPY ./src/ ./src/ # TODO: .git folder is required for building – this is destructive. # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION. @@ -24,13 +21,7 @@ COPY ./videojs-dependencies.yml ./videojs-dependencies.yml RUN crystal spec --warnings all \ --link-flags "-lxml2 -llzma" -RUN if [[ "${release}" == 1 && "${disable_quic}" == 1 ]] ; then \ - crystal build ./src/invidious.cr \ - --release \ - -Ddisable_quic \ - --static --warnings all \ - --link-flags "-lxml2 -llzma"; \ - elif [[ "${release}" == 1 ]] ; then \ +RUN if [[ "${release}" == 1 ]] ; then \ crystal build ./src/invidious.cr \ --release \ --static --warnings all \