Merge branch 'iv-org:master' into main

.github/workflows/ci.yml

@@ -38,10 +38,10 @@ jobs:
       matrix:
         stable: [true]
         crystal:
-          - 1.6.2
           - 1.7.3
           - 1.8.2
           - 1.9.2
+          - 1.10.1
         include:
           - crystal: nightly
             stable: false

.github/workflows/container-release.yml

@@ -57,7 +57,6 @@ jobs:
         with:
           images: quay.io/invidious/invidious
           tags: |
-            type=sha,format=short
             type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
             type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
           labels: |
@@ -83,7 +82,6 @@ jobs:
           flavor: |
             suffix=-arm64
           tags: |
-            type=sha,format=short
             type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
             type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
           labels: |

README.md

@@ -145,18 +145,7 @@ Weblate also allows you to log-in with major SSO providers like Github, Gitlab,
 
 ## Projects using Invidious
 
-- [FreeTube](https://github.com/FreeTubeApp/FreeTube): A libre software YouTube app for privacy.
+A list of projects and extensions for or utilizing Invidious can be found in the documentation: https://docs.invidious.io/applications/
-- [CloudTube](https://sr.ht/~cadence/tube/): A JavaScript-rich alternate YouTube player.
-- [PeerTubeify](https://gitlab.com/Cha_de_L/peertubeify): On YouTube, displays a link to the same video on PeerTube, if it exists.
-- [MusicPiped](https://github.com/deep-gaurav/MusicPiped): A material design music player that streams music from YouTube.
-- [HoloPlay](https://github.com/stephane-r/holoplay-pwa): Progressive Web App connecting on Invidious API's with search, playlists and favorites.
-- [WatchTube](https://github.com/WatchTubeTeam/WatchTube): Powerful YouTube client for Apple Watch.
-- [Yattee](https://github.com/yattee/yattee): Alternative YouTube frontend for iPhone, iPad, Mac and Apple TV.
-- [TubiTui](https://codeberg.org/777/TubiTui): A lightweight, libre, TUI-based YouTube client.
-- [Ytfzf](https://github.com/pystardust/ytfzf): A posix script to find and watch youtube videos from the terminal. (Without API).
-- [Playlet](https://github.com/iBicha/playlet): Unofficial Youtube client for Roku TV.
-- [Clipious](https://github.com/lamarios/clipious): Unofficial Invidious client for Android.
-
 
 ## Liability
 

assets/js/player.js

@@ -752,6 +752,17 @@ if (navigator.vendor === 'Apple Computer, Inc.' && video_data.params.listen) {
     });
 }
 
+// Safari screen timeout on looped video playback fix
+if (navigator.vendor === 'Apple Computer, Inc.' && !video_data.params.listen && video_data.params.video_loop) {
+    player.loop(false);
+    player.ready(function () {
+        player.on('ended', function () {
+            player.currentTime(0);
+            player.play();
+        });
+    });
+}
+
 // Watch on Invidious link
 if (location.pathname.startsWith('/embed/')) {
     const Button = videojs.getComponent('Button');

config/config.example.yml

@@ -392,27 +392,6 @@ jobs:
     enable: true
 
 
-# -----------------------------
-#  Captcha API
-# -----------------------------
-
-##
-## URL of the captcha solving service.
-##
-## Accepted values: any URL
-## Default: https://api.anti-captcha.com
-##
-#captcha_api_url: https://api.anti-captcha.com
-
-##
-## API key for the captcha solving service.
-##
-## Accepted values: a string
-## Default: <none>
-##
-#captcha_key:
-
-
 # -----------------------------
 #  Miscellaneous
 # -----------------------------

spec/invidious/helpers_spec.cr

@@ -3,18 +3,6 @@ require "../spec_helper"
 CONFIG = Config.from_yaml(File.open("config/config.example.yml"))
 
 Spectator.describe "Helper" do
-  describe "#produce_channel_videos_url" do
-    it "correctly produces url for requesting page `x` of a channel's videos" do
-      # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw")).to eq("/browse_ajax?continuation=4qmFsgI8EhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0V4&gl=US&hl=en")
-      #
-      # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw", sort_by: "popular")).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0V4R0FFPQ%3D%3D&gl=US&hl=en")
-
-      # expect(produce_channel_videos_url(ucid: "UCXuqSBlHAE6Xw-yeJA0Tunw", page: 20)).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0l5TUE9PQ%3D%3D&gl=US&hl=en")
-
-      # expect(produce_channel_videos_url(ucid: "UC-9-kyTW8ZkZNDHQJ6FgpwQ", page: 20, sort_by: "popular")).to eq("/browse_ajax?continuation=4qmFsgJAEhhVQy05LWt5VFc4WmtaTkRIUUo2Rmdwd1EaJEVnWjJhV1JsYjNNd0FqZ0JZQUZxQUxnQkFDQUFlZ0l5TUJnQg%3D%3D&gl=US&hl=en")
-    end
-  end
-
   describe "#produce_channel_search_continuation" do
     it "correctly produces token for searching a specific channel" do
       expect(produce_channel_search_continuation("UCXuqSBlHAE6Xw-yeJA0Tunw", "", 100)).to eq("4qmFsgJqEhhVQ1h1cVNCbEhBRTZYdy15ZUpBMFR1bncaIEVnWnpaV0Z5WTJnd0FUZ0JZQUY2QkVkS2IxaTRBUUE9WgCaAilicm93c2UtZmVlZFVDWHVxU0JsSEFFNlh3LXllSkEwVHVud3NlYXJjaA%3D%3D")

src/invidious/channels/channels.cr

@@ -93,7 +93,7 @@ struct ChannelVideo
   def to_tuple
     {% begin %}
       {
-        {{*@type.instance_vars.map(&.name)}}
+        {{@type.instance_vars.map(&.name).splat}}
       }
     {% end %}
   end

src/invidious/channels/videos.cr

@@ -62,12 +62,6 @@ def produce_channel_videos_continuation(ucid, page = 1, auto_generated = nil, so
   return continuation
 end
 
-# Used in bypass_captcha_job.cr
-def produce_channel_videos_url(ucid, page = 1, auto_generated = nil, sort_by = "newest", v2 = false)
-  continuation = produce_channel_videos_continuation(ucid, page, auto_generated, sort_by, v2)
-  return "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
-end
-
 module Invidious::Channel::Tabs
   extend self
 

src/invidious/config.cr

@@ -48,7 +48,7 @@ struct ConfigPreferences
   def to_tuple
     {% begin %}
       {
-        {{*@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }}}
+        {{(@type.instance_vars.map { |var| "#{var.name}: #{var.name}".id }).splat}}
       }
     {% end %}
   end
@@ -133,10 +133,6 @@ class Config
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new
-  # Key for Anti-Captcha
-  property captcha_key : String? = nil
-  # API URL for Anti-Captcha
-  property captcha_api_url : String = "https://api.anti-captcha.com"
 
   # Playlist length limit
   property playlist_length_limit : Int32 = 500

src/invidious/helpers/errors.cr

@@ -3,7 +3,7 @@
 # -------------------
 
 macro error_template(*args)
-  error_template_helper(env, {{*args}})
+  error_template_helper(env, {{args.splat}})
 end
 
 def github_details(summary : String, content : String)
@@ -95,7 +95,7 @@ end
 # -------------------
 
 macro error_atom(*args)
-  error_atom_helper(env, {{*args}})
+  error_atom_helper(env, {{args.splat}})
 end
 
 def error_atom_helper(env : HTTP::Server::Context, status_code : Int32, exception : Exception)
@@ -121,7 +121,7 @@ end
 # -------------------
 
 macro error_json(*args)
-  error_json_helper(env, {{*args}})
+  error_json_helper(env, {{args.splat}})
 end
 
 def error_json_helper(

src/invidious/jobs/bypass_captcha_job.cr

@@ -1,135 +0,0 @@
-class Invidious::Jobs::BypassCaptchaJob < Invidious::Jobs::BaseJob
-  def begin
-    loop do
-      begin
-        random_video = PG_DB.query_one?("select id, ucid from (select id, ucid from channel_videos limit 1000) as s ORDER BY RANDOM() LIMIT 1", as: {id: String, ucid: String})
-        if !random_video
-          random_video = {id: "zj82_v2R6ts", ucid: "UCK87Lox575O_HCHBWaBSyGA"}
-        end
-        {"/watch?v=#{random_video["id"]}&gl=US&hl=en&has_verified=1&bpctr=9999999999", produce_channel_videos_url(ucid: random_video["ucid"])}.each do |path|
-          response = YT_POOL.client &.get(path)
-          if response.body.includes?("To continue with your YouTube experience, please fill out the form below.")
-            html = XML.parse_html(response.body)
-            form = html.xpath_node(%(//form[@action="/das_captcha"])).not_nil!
-            site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
-            s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
-
-            inputs = {} of String => String
-            form.xpath_nodes(%(.//input[@name])).map do |node|
-              inputs[node["name"]] = node["value"]
-            end
-
-            headers = response.cookies.add_request_headers(HTTP::Headers.new)
-
-            response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/createTask",
-              headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
-              "clientKey" => CONFIG.captcha_key,
-              "task"      => {
-                "type"                => "NoCaptchaTaskProxyless",
-                "websiteURL"          => "https://www.youtube.com#{path}",
-                "websiteKey"          => site_key,
-                "recaptchaDataSValue" => s_value,
-              },
-            }.to_json).body)
-
-            raise response["error"].as_s if response["error"]?
-            task_id = response["taskId"].as_i
-
-            loop do
-              sleep 10.seconds
-
-              response = JSON.parse(HTTP::Client.post(CONFIG.captcha_api_url + "/getTaskResult",
-                headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
-                "clientKey" => CONFIG.captcha_key,
-                "taskId"    => task_id,
-              }.to_json).body)
-
-              if response["status"]?.try &.== "ready"
-                break
-              elsif response["errorId"]?.try &.as_i != 0
-                raise response["errorDescription"].as_s
-              end
-            end
-
-            inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
-            headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
-            response = YT_POOL.client &.post("/das_captcha", headers, form: inputs)
-
-            response.cookies
-              .select { |cookie| cookie.name != "PREF" }
-              .each { |cookie| CONFIG.cookies << cookie }
-
-            # Persist cookies between runs
-            File.write("config/config.yml", CONFIG.to_yaml)
-          elsif response.headers["Location"]?.try &.includes?("/sorry/index")
-            location = response.headers["Location"].try { |u| URI.parse(u) }
-            headers = HTTP::Headers{":authority" => location.host.not_nil!}
-            response = YT_POOL.client &.get(location.request_target, headers)
-
-            html = XML.parse_html(response.body)
-            form = html.xpath_node(%(//form[@action="index"])).not_nil!
-            site_key = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-sitekey"]
-            s_value = form.xpath_node(%(.//div[@id="recaptcha"])).try &.["data-s"]
-
-            inputs = {} of String => String
-            form.xpath_nodes(%(.//input[@name])).map do |node|
-              inputs[node["name"]] = node["value"]
-            end
-
-            captcha_client = HTTPClient.new(URI.parse(CONFIG.captcha_api_url))
-            captcha_client.family = CONFIG.force_resolve || Socket::Family::INET
-            response = JSON.parse(captcha_client.post("/createTask",
-              headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
-              "clientKey" => CONFIG.captcha_key,
-              "task"      => {
-                "type"                => "NoCaptchaTaskProxyless",
-                "websiteURL"          => location.to_s,
-                "websiteKey"          => site_key,
-                "recaptchaDataSValue" => s_value,
-              },
-            }.to_json).body)
-
-            captcha_client.close
-
-            raise response["error"].as_s if response["error"]?
-            task_id = response["taskId"].as_i
-
-            loop do
-              sleep 10.seconds
-
-              response = JSON.parse(captcha_client.post("/getTaskResult",
-                headers: HTTP::Headers{"Content-Type" => "application/json"}, body: {
-                "clientKey" => CONFIG.captcha_key,
-                "taskId"    => task_id,
-              }.to_json).body)
-
-              if response["status"]?.try &.== "ready"
-                break
-              elsif response["errorId"]?.try &.as_i != 0
-                raise response["errorDescription"].as_s
-              end
-            end
-
-            inputs["g-recaptcha-response"] = response["solution"]["gRecaptchaResponse"].as_s
-            headers["Cookies"] = response["solution"]["cookies"].as_h?.try &.map { |k, v| "#{k}=#{v}" }.join("; ") || ""
-            response = YT_POOL.client &.post("/sorry/index", headers: headers, form: inputs)
-            headers = HTTP::Headers{
-              "Cookie" => URI.parse(response.headers["location"]).query_params["google_abuse"].split(";")[0],
-            }
-            cookies = HTTP::Cookies.from_client_headers(headers)
-
-            cookies.each { |cookie| CONFIG.cookies << cookie }
-
-            # Persist cookies between runs
-            File.write("config/config.yml", CONFIG.to_yaml)
-          end
-        end
-      rescue ex
-        LOGGER.error("BypassCaptchaJob: #{ex.message}")
-      ensure
-        sleep 1.minute
-        Fiber.yield
-      end
-    end
-  end
-end

src/invidious/videos.cr

@@ -227,8 +227,22 @@ struct Video
     info.dig?("streamingData", "hlsManifestUrl").try &.as_s
   end
 
-  def dash_manifest_url
+  def dash_manifest_url : String?
-    info.dig?("streamingData", "dashManifestUrl").try &.as_s
+    raw_dash_url = info.dig?("streamingData", "dashManifestUrl").try &.as_s
+    return nil if raw_dash_url.nil?
+
+    # Use manifest v5 parameter to reduce file size
+    # See https://github.com/iv-org/invidious/issues/4186
+    dash_url = URI.parse(raw_dash_url)
+    dash_query = dash_url.query || ""
+
+    if dash_query.empty?
+      dash_url.path = "#{dash_url.path}/mpd_version/5"
+    else
+      dash_url.query = "#{dash_query}&mpd_version=5"
+    end
+
+    return dash_url.to_s
   end
 
   def genre_url : String?

src/invidious/yt_backend/connection_pool.cr

@@ -1,7 +1,6 @@
 def add_yt_headers(request)
-  if request.headers["User-Agent"] == "Crystal"
+  request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
-    request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
+  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
-  end
 
   request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
   request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"