Bump version

Fixed double quotes in meta description

docker/Dockerfile

@@ -1,22 +1,28 @@
-FROM alpine:edge AS builder
+FROM alpine:edge
 RUN apk add --no-cache crystal shards libc-dev \
-    yaml-dev libxml2-dev sqlite-dev zlib-dev openssl-dev \
-    sqlite-static zlib-static openssl-libs-static
+    yaml-dev libxml2-dev sqlite-dev zlib-dev curl && \
+    curl -Lo /etc/apk/keys/omarroth.rsa.pub https://github.com/omarroth/boringssl-alpine/releases/download/1.1.0-r0/omarroth.rsa.pub && \
+    curl -Lo boringssl-dev.apk https://github.com/omarroth/boringssl-alpine/releases/download/1.1.0-r0/boringssl-dev-1.1.0-r0.apk && \
+    curl -Lo lsquic.apk https://github.com/omarroth/lsquic-alpine/releases/download/2.6.3-r0/lsquic-2.6.3-r0.apk && \
+    apk update && \
+    apk add boringssl-dev.apk lsquic.apk && \
+    rm -rf /var/cache/apk/* boringssl-dev.apk lsquic.apk
 WORKDIR /invidious
 COPY ./shard.yml ./shard.yml
 RUN shards update && shards install
+RUN cp /usr/lib/libcrypto.a ./lib/lsquic/src/lsquic/ext/libcrypto.a && \
+    cp /usr/lib/libssl.a ./lib/lsquic/src/lsquic/ext/libssl.a && \
+    cp /usr/lib/liblsquic.a ./lib/lsquic/src/lsquic/ext/liblsquic.a
 COPY ./src/ ./src/
 # TODO: .git folder is required for building – this is destructive.
 # See definition of CURRENT_BRANCH, CURRENT_COMMIT and CURRENT_VERSION.
 COPY ./.git/ ./.git/
-RUN crystal build --static --release --warnings all --error-on-warnings \
-# TODO: Remove next line, see https://github.com/crystal-lang/crystal/issues/7946
+RUN crystal build --release --warnings all --error-on-warnings \
+    # TODO: Remove next line, see https://github.com/crystal-lang/crystal/issues/7946
     -Dmusl \
     ./src/invidious.cr
 
-FROM alpine:latest
 RUN apk add --no-cache librsvg ttf-opensans
-WORKDIR /invidious
 RUN addgroup -g 1000 -S invidious && \
     adduser -u 1000 -S invidious -G invidious
 COPY ./assets/ ./assets/
@@ -24,6 +30,5 @@ COPY ./config/config.yml ./config/config.yml
 COPY ./config/sql/ ./config/sql/
 COPY ./locales/ ./locales/
 RUN sed -i 's/host: \(127.0.0.1\|localhost\)/host: postgres/' config/config.yml
-COPY --from=builder /invidious/invidious .
 USER invidious
 CMD [ "/invidious/invidious" ]

shard.yml

@@ -1,5 +1,5 @@
 name: invidious
-version: 0.20.0
+version: 0.20.1
 
 authors:
   - Omar Roth <omarroth@protonmail.com>
@@ -24,6 +24,9 @@ dependencies:
   protodec:
     github: omarroth/protodec
     version: ~> 0.1.2
+  lsquic:
+    github: omarroth/lsquic.cr
+    version: ~> 0.1.3
 
 crystal: 0.31.1
 

spec/helpers_spec.cr

@@ -34,7 +34,7 @@ describe "Helper" do
 
   describe "#produce_channel_playlists_url" do
     it "correctly produces a /browse_ajax URL with the given UCID and cursor" do
-      produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW").should eq("/browse_ajax?continuation=4qmFsgLJARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GqwBRWdod2JHRjViR2x6ZERBQ09BRmdBV29BdUFFQWVtWlJWV3hRWVRGck5WSldSbmRoVmpsdVpWYzBlRmd4Um5sU2JsWmhUVmhLYkZScVozaFlNREZPWWxWcmVGZFhNWE5SYlVwelpIcG9jVTR3Y0VsVFdGSkdVbXRqTVdGRVpIaFpNSEJWVkcxUk1GWjZiRFJPVmtZeFlURTVSRlpzYjNkTmFtaHVWbmNnQVJnRA%3D%3D&gl=US&hl=en")
+      produce_channel_playlists_url("UCCj956IF62FbT7Gouszaj9w", "AIOkY9EQpi_gyn1_QrFuZ1reN81_MMmI1YmlBblw8j7JHItEFG5h7qcJTNd4W9x5Quk_CVZ028gW").should eq("/browse_ajax?continuation=4qmFsgLNARIYVUNDajk1NklGNjJGYlQ3R291c3phajl3GrABRWdsd2JHRjViR2x6ZEhNd0FqZ0JZQUZxQUxnQkFIcG1VVlZzVUdFeGF6VlNWa1ozWVZZNWJtVlhOSGhZTVVaNVVtNVdZVTFZU214VWFtZDRXREF4VG1KVmEzaFhWekZ6VVcxS2MyUjZhSEZPTUhCSlUxaFNSbEpyWXpGaFJHUjRXVEJ3VlZSdFVUQldlbXcwVGxaR01XRXhPVVJXYkc5M1RXcG9ibFozSUFFWUF3PT0%3D&gl=US&hl=en")
     end
   end
 

src/invidious.cr

@@ -94,7 +94,7 @@ LOCALES = {
   "zh-TW" => load_locale("zh-TW"),
 }
 
-YT_POOL     = HTTPPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
+YT_POOL     = QUICPool.new(YT_URL, capacity: CONFIG.pool_size, timeout: 0.05)
 YT_IMG_POOL = HTTPPool.new(YT_IMG_URL, capacity: CONFIG.pool_size, timeout: 0.05)
 
 config = CONFIG
@@ -212,6 +212,20 @@ spawn do
   end
 end
 
+if CONFIG.captcha_key
+  spawn do
+    bypass_captcha(CONFIG.captcha_key, logger) do |cookies|
+      cookies.each do |cookie|
+        config.cookies << cookie
+      end
+
+      # Persist cookies between runs
+      CONFIG.cookies = config.cookies
+      File.write("config/config.yml", config.to_yaml)
+    end
+  end
+end
+
 connection_channel = Channel({Bool, Channel(PQ::Notification)}).new(32)
 spawn do
   connections = [] of Channel(PQ::Notification)
@@ -1421,6 +1435,7 @@ post "/login" do |env|
     traceback = IO::Memory.new
 
     # See https://github.com/ytdl-org/youtube-dl/blob/2019.04.07/youtube_dl/extractor/youtube.py#L82
+    # TODO: Convert to QUIC
     begin
       client = make_client(LOGIN_URL)
       headers = HTTP::Headers.new
@@ -1445,7 +1460,7 @@ post "/login" do |env|
 
       headers["Content-Type"] = "application/x-www-form-urlencoded;charset=utf-8"
       headers["Google-Accounts-XSRF"] = "1"
-      headers["User-Agent"] = random_user_agent
+      headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
 
       response = client.post("/_/signin/sl/lookup", headers, login_req(lookup_req))
       lookup_results = JSON.parse(response.body[5..-1])
@@ -4029,7 +4044,13 @@ get "/api/v1/annotations/:id" do |env|
     annotations = response.body
   end
 
-  annotations
+  etag = sha256(annotations)[0, 16]
+  if env.request.headers["If-None-Match"]?.try &.== etag
+    env.response.status_code = 304
+  else
+    env.response.headers["ETag"] = etag
+    annotations
+  end
 end
 
 get "/api/v1/videos/:id" do |env|
@@ -4489,8 +4510,9 @@ get "/api/v1/search/suggestions" do |env|
   query ||= ""
 
   begin
-    client = make_client(URI.parse("https://suggestqueries.google.com"))
-    response = client.get("/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback").body
+    response = QUIC::Client.get(
+      "https://suggestqueries.google.com/complete/search?hl=en&gl=#{region}&client=youtube&ds=yt&q=#{URI.encode_www_form(query)}&callback=suggestCallback"
+    ).body
 
     body = response[35..-2]
     body = JSON.parse(body).as_a

src/invidious/channels.cr

@@ -484,7 +484,7 @@ def produce_channel_playlists_url(ucid, cursor, sort = "newest", auto_generated
     "80226972:embedded" => {
       "2:string" => ucid,
       "3:base64" => {
-        "2:string" => "playlist",
+        "2:string" => "playlists",
         "6:varint":   2_i64,
         "7:varint":   1_i64,
         "12:varint":  1_i64,

src/invidious/helpers/helpers.cr

@@ -192,6 +192,27 @@ struct Config
     end
   end
 
+  module StringToCookies
+    def self.to_yaml(value : HTTP::Cookies, yaml : YAML::Nodes::Builder)
+      (value.map { |c| "#{c.name}=#{c.value}" }).join("; ").to_yaml(yaml)
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : HTTP::Cookies
+      unless node.is_a?(YAML::Nodes::Scalar)
+        node.raise "Expected scalar, not #{node.class}"
+      end
+
+      cookies = HTTP::Cookies.new
+      node.value.split(";").each do |cookie|
+        next if cookie.strip.empty?
+        name, value = cookie.split("=", 2)
+        cookies << HTTP::Cookie.new(name.strip, value.strip)
+      end
+
+      cookies
+    end
+  end
+
   def disabled?(option)
     case disabled = CONFIG.disable_proxy
     when Bool
@@ -236,6 +257,8 @@ struct Config
     host_binding:      {type: String, default: "0.0.0.0"},                                                  # Host to bind (overrided by command line argument)
     pool_size:         {type: Int32, default: 100},                                                         # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
     admin_email:       {type: String, default: "omarroth@protonmail.com"},                                  # Email for bug reports
+    cookies:           {type: HTTP::Cookies, default: HTTP::Cookies.new, converter: StringToCookies},       # Saved cookies in "name1=value1; name2=value2..." format
+    captcha_key:       {type: String?, default: nil},                                                       # Key for Anti-Captcha
   })
 end
 

src/invidious/helpers/jobs.cr

@@ -228,10 +228,77 @@ def update_decrypt_function
       yield decrypt_function
     rescue ex
       next
+    ensure
+      sleep 1.minute
+      Fiber.yield
     end
+  end
+end
 
-    sleep 1.minute
-    Fiber.yield
+def bypass_captcha(captcha_key, logger)
+  loop do
+    begin
+      response = YT_POOL.client &.get("/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999")
+      if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
+        html = XML.parse_html(response.body)
+        form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
+        site_key = form.xpath_node(%(.//div[@class="g-recaptcha"])).try &.["data-sitekey"]
+
+        inputs = {} of String => String
+        form.xpath_nodes(%(.//input[@name])).map do |node|
+          inputs[node["name"]] = node["value"]
+        end
+
+        headers = response.cookies.add_request_headers(HTTP::Headers.new)
+
+        response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/createTask", body: {
+          "clientKey" => CONFIG.captcha_key,
+          "task"      => {
+            "type" => "NoCaptchaTaskProxyless",
+            # "type"          => "NoCaptchaTask",
+            "websiteURL" => "https://www.youtube.com/watch?v=CvFH_6DNRCY&gl=US&hl=en&disable_polymer=1&has_verified=1&bpctr=9999999999",
+            "websiteKey" => site_key,
+            # "proxyType"     => "http",
+            # "proxyAddress"  => CONFIG.proxy_address,
+            # "proxyPort"     => CONFIG.proxy_port,
+            # "proxyLogin"    => CONFIG.proxy_user,
+            # "proxyPassword" => CONFIG.proxy_pass,
+            # "userAgent"     => "User-Agent: Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.97 Safari/537.36",
+          },
+        }.to_json).body)
+
+        if response["error"]?
+          raise response["error"].as_s
+        end
+
+        task_id = response["taskId"].as_i
+
+        loop do
+          sleep 10.seconds
+
+          response = JSON.parse(HTTP::Client.post("https://api.anti-captcha.com/getTaskResult", body: {
+            "clientKey" => CONFIG.captcha_key,
+            "taskId"    => task_id,
+          }.to_json).body)
+
+          if response["status"]?.try &.== "ready"
+            break
+          elsif response["errorId"]?.try &.as_i != 0
+            raise response["errorDescription"].as_s
+          end
+        end
+
+        inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
+        response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
+
+        yield response.cookies.select { |cookie| cookie.name != "PREF" }
+      end
+    rescue ex
+      logger.puts("Exception: #{ex.message}")
+    ensure
+      sleep 1.minute
+      Fiber.yield
+    end
   end
 end
 

src/invidious/helpers/proxy.cr

@@ -90,19 +90,6 @@ class HTTPClient < HTTP::Client
 
     return opts
   end
-
-  def exec(request)
-    if self.host == "www.youtube.com"
-      request.headers["x-youtube-client-name"] ||= "1"
-      request.headers["x-youtube-client-version"] ||= "1.20180719"
-      request.headers["user-agent"] ||= random_user_agent
-      request.headers["accept-charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
-      request.headers["accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-      request.headers["accept-language"] ||= "en-us,en;q=0.5"
-    end
-
-    super
-  end
 end
 
 def get_proxies(country_code = "US")
@@ -114,6 +101,7 @@ def filter_proxies(proxies)
   proxies.select! do |proxy|
     begin
       client = HTTPClient.new(YT_URL)
+      client.before_request { |r| add_yt_headers(r) } if url.host == "www.youtube.com"
       client.read_timeout = 10.seconds
       client.connect_timeout = 10.seconds
 

src/invidious/search.cr

@@ -333,7 +333,7 @@ def produce_search_params(sort : String = "relevance", date : String = "", conte
   when "short"
     object["2:embedded"].as(Hash)["3:varint"] = 1_i64
   when "long"
-    object["2:embedded"].as(Hash)["3:varint"] = 18_i64
+    object["2:embedded"].as(Hash)["3:varint"] = 2_i64
   end
 
   features.each do |feature|

src/invidious/views/watch.ecr

@@ -1,12 +1,12 @@
 <% content_for "header" do %>
 <meta name="thumbnail" content="<%= thumbnail %>">
-<meta name="description" content="<%= video.short_description %>">
+<meta name="description" content="<%= HTML.escape(video.short_description) %>">
 <meta name="keywords" content="<%= video.keywords.join(",") %>">
 <meta property="og:site_name" content="Invidious">
 <meta property="og:url" content="<%= host_url %>/watch?v=<%= video.id %>">
 <meta property="og:title" content="<%= HTML.escape(video.title) %>">
 <meta property="og:image" content="/vi/<%= video.id %>/maxres.jpg">
-<meta property="og:description" content="<%= video.short_description %>">
+<meta property="og:description" content="<%= HTML.escape(video.short_description) %>">
 <meta property="og:type" content="video.other">
 <meta property="og:video:url" content="<%= host_url %>/embed/<%= video.id %>">
 <meta property="og:video:secure_url" content="<%= host_url %>/embed/<%= video.id %>">
@@ -17,7 +17,7 @@
 <meta name="twitter:site" content="@omarroth1">
 <meta name="twitter:url" content="<%= host_url %>/watch?v=<%= video.id %>">
 <meta name="twitter:title" content="<%= HTML.escape(video.title) %>">
-<meta name="twitter:description" content="<%= video.short_description %>">
+<meta name="twitter:description" content="<%= HTML.escape(video.short_description) %>">
 <meta name="twitter:image" content="<%= host_url %>/vi/<%= video.id %>/maxres.jpg">
 <meta name="twitter:player" content="<%= host_url %>/embed/<%= video.id %>">
 <meta name="twitter:player:width" content="1280">