Merge 734e72503f into 9892604758

config/config.example.yml

@@ -54,6 +54,32 @@ db:
 ##
 #signature_server:
 
+##
+## Invidious companion is an external program
+## for loading the video streams from YouTube servers.
+##
+## When this setting is commented out, Invidious companion is not used.
+##
+## When this setting is configured, then Invidious will proxy the requests
+## to Invidious companion.
+## Or randomly choose one if multiple Invidious companion are configured.
+##
+## Accepted values: "http(s)://<IP-HOSTNAME>:<Port>"
+## Default: <none>
+##
+#invidious_companion:
+# - http://127.0.0.1:8282
+
+##
+## API key for Invidious companion
+##
+## Needed when invidious_companion is configured
+##
+## Accepted values: "http(s)://<IP-HOSTNAME>:<Port>"
+## Default: <none>
+##
+
+#invidious_companion_key: "CHANGE_ME!!"
 
 #########################################
 #

src/invidious/config.cr

@@ -67,6 +67,28 @@ end
 class Config
   include YAML::Serializable
 
+  module URIArrayConverter
+    def self.to_yaml(values : Array(URI), yaml : YAML::Nodes::Builder)
+      yaml.sequence do
+        values.each { |v| yaml.scalar v.to_s }
+      end
+    end
+
+    def self.from_yaml(ctx : YAML::ParseContext, node : YAML::Nodes::Node) : Array(URI)
+      if node.is_a?(YAML::Nodes::Sequence)
+        node.map do |child|
+          unless child.is_a?(YAML::Nodes::Scalar)
+            node.raise "Expected scalar, not #{child.class}"
+          end
+
+          URI.parse(child.value)
+        end
+      else
+        node.raise "Expected sequence, not #{node.class}"
+      end
+    end
+  end
+
   # Number of threads to use for crawling videos from channels (for updating subscriptions)
   property channel_threads : Int32 = 1
   # Time interval between two executions of the job that crawls channel videos (subscriptions update).
@@ -151,6 +173,13 @@ class Config
   # poToken for passing bot attestation
   property po_token : String? = nil
 
+  # Invidious companion
+  @[YAML::Field(converter: Config::URIArrayConverter)]
+  property invidious_companion : Array(URI) = [] of URI
+
+  # Invidious companion API key
+  property invidious_companion_key : String = ""
+
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new
@@ -222,6 +251,21 @@ class Config
         end
     {% end %}
 
+    if !config.invidious_companion.empty?
+      # invidious_companion and signature_server can't work together
+      if config.signature_server
+        puts "Config: You can not run inv_sig_helper and invidious_companion at the same time."
+        exit(1)
+      end
+      if config.invidious_companion_key.empty?
+        puts "Config: Please configure a key if you are using invidious companion."
+        exit(1)
+      elsif config.invidious_companion_key == "CHANGE_ME!!"
+        puts "Config: The value of 'invidious_companion_key' needs to be changed!!"
+        exit(1)
+      end
+    end
+
     # HMAC_key is mandatory
     # See: https://github.com/iv-org/invidious/issues/3854
     if config.hmac_key.empty?

src/invidious/routes/api/manifest.cr

@@ -1,6 +1,10 @@
 module Invidious::Routes::API::Manifest
   # /api/manifest/dash/id/:id
   def self.get_dash_video_id(env)
+    if !CONFIG.invidious_companion.empty?
+      return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.")
+    end
+
     env.response.headers.add("Access-Control-Allow-Origin", "*")
     env.response.content_type = "application/dash+xml"
 

src/invidious/routes/embed.cr

@@ -201,6 +201,13 @@ module Invidious::Routes::Embed
       return env.redirect url
     end
 
+    if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash"))
+      env.response.headers["Content-Security-Policy"] =
+        env.response.headers["Content-Security-Policy"]
+          .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
+          .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
+    end
+
     rendered "embed"
   end
 end

src/invidious/routes/video_playback.cr

@@ -253,6 +253,10 @@ module Invidious::Routes::VideoPlayback
   # YouTube /videoplayback links expire after 6 hours,
   # so we have a mechanism here to redirect to the latest version
   def self.latest_version(env)
+    if !CONFIG.invidious_companion.empty?
+      return error_template(403, "This endpoint is not permitted because it is handled by Invidious companion.")
+    end
+
     id = env.params.query["id"]?
     itag = env.params.query["itag"]?.try &.to_i?
 

src/invidious/routes/watch.cr

@@ -190,6 +190,13 @@ module Invidious::Routes::Watch
       captions: video.captions
     )
 
+    if (!CONFIG.invidious_companion.empty? && (preferences.local || preferences.quality == "dash"))
+      env.response.headers["Content-Security-Policy"] =
+        env.response.headers["Content-Security-Policy"]
+          .gsub("media-src", "media-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
+          .gsub("connect-src", "connect-src " + video.invidious_companion.not_nil!["baseUrl"].as_s)
+    end
+
     templated "watch"
   end
 
@@ -321,13 +328,20 @@ module Invidious::Routes::Watch
 
       return Invidious::Routes::API::V1::Videos.captions(env)
     elsif itag = download_widget["itag"]?.try &.as_i
+      itag = itag.to_s
+
       # URL params specific to /latest_version
       env.params.query["id"] = video_id
-      env.params.query["itag"] = itag.to_s
+      env.params.query["itag"] = itag
       env.params.query["title"] = filename
       env.params.query["local"] = "true"
 
-      return Invidious::Routes::VideoPlayback.latest_version(env)
+      if (!CONFIG.invidious_companion.empty?)
+        video = get_video(video_id)
+        return env.redirect "#{video.invidious_companion.not_nil!["baseUrl"].as_s}/latest_version?id=#{video_id}&itag=#{itag}&local=true"
+      else
+        return Invidious::Routes::VideoPlayback.latest_version(env)
+      end
     else
       return error_template(400, "Invalid label or itag")
     end

src/invidious/videos.cr

@@ -15,7 +15,7 @@ struct Video
   # NOTE: don't forget to bump this number if any change is made to
   # the `params` structure in videos/parser.cr!!!
   #
-  SCHEMA_VERSION = 2
+  SCHEMA_VERSION = 3
 
   property id : String
 
@@ -192,6 +192,10 @@ struct Video
     }
   end
 
+  def invidious_companion : Hash(String, JSON::Any)?
+    info["invidiousCompanion"]?.try &.as_h
+  end
+
   # Macros defining getters/setters for various types of data
 
   private macro getset_string(name)

src/invidious/videos/parser.cr

@@ -100,30 +100,32 @@ def extract_video_info(video_id : String)
   params = parse_video_info(video_id, player_response)
   params["reason"] = JSON::Any.new(reason) if reason
 
-  new_player_response = nil
+  if !CONFIG.invidious_companion.empty?
+    new_player_response = nil
 
-  # Don't use Android test suite client if po_token is passed because po_token doesn't
-  # work for Android test suite client.
-  if reason.nil? && CONFIG.po_token.nil?
-    # Fetch the video streams using an Android client in order to get the
-    # decrypted URLs and maybe fix throttling issues (#2194). See the
-    # following issue for an explanation about decrypted URLs:
-    # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
-    client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
-    new_player_response = try_fetch_streaming_data(video_id, client_config)
+    # Don't use Android test suite client if po_token is passed because po_token doesn't
+    # work for Android test suite client.
+    if reason.nil? && CONFIG.po_token.nil?
+      # Fetch the video streams using an Android client in order to get the
+      # decrypted URLs and maybe fix throttling issues (#2194). See the
+      # following issue for an explanation about decrypted URLs:
+      # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562
+      client_config.client_type = YoutubeAPI::ClientType::AndroidTestSuite
+      new_player_response = try_fetch_streaming_data(video_id, client_config)
+    end
+
+    # Replace player response and reset reason
+    if !new_player_response.nil?
+      # Preserve captions & storyboard data before replacement
+      new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
+      new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
+
+      player_response = new_player_response
+      params.delete("reason")
+    end
   end
 
-  # Replace player response and reset reason
-  if !new_player_response.nil?
-    # Preserve captions & storyboard data before replacement
-    new_player_response["storyboards"] = player_response["storyboards"] if player_response["storyboards"]?
-    new_player_response["captions"] = player_response["captions"] if player_response["captions"]?
-
-    player_response = new_player_response
-    params.delete("reason")
-  end
-
-  {"captions", "playabilityStatus", "playerConfig", "storyboards"}.each do |f|
+  {"captions", "playabilityStatus", "playerConfig", "storyboards", "invidiousCompanion"}.each do |f|
     params[f] = player_response[f] if player_response[f]?
   end
 

src/invidious/views/components/player.ecr

@@ -22,6 +22,7 @@
                audio_streams.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local)
 
                 bitrate = fmt["bitrate"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)
@@ -34,8 +35,11 @@
                 <% end %>
             <% end %>
         <% else %>
-            <% if params.quality == "dash" %>
-                <source src="/api/manifest/dash/id/<%= video.id %>?local=true&unique_res=1" type='application/dash+xml' label="dash">
+            <% if params.quality == "dash"
+               src_url = "/api/manifest/dash/id/" + video.id + "?local=true&unique_res=1"
+               src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty?)
+            %>
+                <source src="<%= src_url %>" type='application/dash+xml' label="dash">
             <% end %>
 
             <%
@@ -44,6 +48,7 @@
             fmt_stream.each_with_index do |fmt, i|
                 src_url  = "/latest_version?id=#{video.id}&itag=#{fmt["itag"]}"
                 src_url += "&local=true" if params.local
+                src_url = video.invidious_companion.not_nil!["baseUrl"].as_s + src_url if (!CONFIG.invidious_companion.empty? && params.local)
 
                 quality = fmt["quality"]
                 mimetype = HTML.escape(fmt["mimeType"].as_s)

src/invidious/yt_backend/youtube_api.cr

@@ -500,7 +500,11 @@ module YoutubeAPI
       data["params"] = params
     end
 
-    return self._post_json("/youtubei/v1/player", data, client_config)
+    if !CONFIG.invidious_companion.empty?
+      return self._post_invidious_companion("/youtubei/v1/player", data)
+    else
+      return self._post_json("/youtubei/v1/player", data, client_config)
+    end
   end
 
   ####################################################################
@@ -666,6 +670,52 @@ module YoutubeAPI
     return initial_data
   end
 
+  ####################################################################
+  # _post_invidious_companion(endpoint, data)
+  #
+  # Internal function that does the actual request to Invidious companion
+  # and handles errors.
+  #
+  # The requested data is an endpoint (URL without the domain part)
+  # and the data as a Hash object.
+  #
+  def _post_invidious_companion(
+    endpoint : String,
+    data : Hash
+  ) : Hash(String, JSON::Any)
+    headers = HTTP::Headers{
+      "Content-Type"    => "application/json; charset=UTF-8",
+      "Authorization"   => "Bearer " + CONFIG.invidious_companion_key,
+    }
+
+    # Logging
+    LOGGER.debug("Invidious companion: Using endpoint: \"#{endpoint}\"")
+    LOGGER.trace("Invidious companion: POST data: #{data}")
+
+    # Send the POST request
+
+    begin
+      response = make_client(CONFIG.invidious_companion.sample,
+        &.post(endpoint, headers: headers, body: data.to_json))
+      body = response.body
+      if (response.status_code != 200)
+        raise Exception.new("Error while communicating with Invidious companion: \
+                              status code: " + response.status_code.to_s + " and body: " + body)
+      end
+    rescue ex
+      raise InfoException.new("Error while communicating with Invidious companion: " + (ex.message || "no extra info found"))
+    end
+
+    if body.nil?
+      raise InfoException.new("Error while communicating with Invidious companion: no response data.")
+    end
+
+    # Convert result to Hash
+    initial_data = JSON.parse(body).as_h
+
+    return initial_data
+  end
+
   ####################################################################
   # _decompress(body_io, headers)
   #