Captions: Add ability to use Innertube's transcripts API (#4001)

config/config.example.yml

@@ -161,6 +161,19 @@ https_only: false
 #force_resolve:
 
 
+##
+## Use Innertube's transcripts API instead of timedtext for closed captions
+##
+## Useful for larger instances as InnerTube is **not ratelimited**. See https://github.com/iv-org/invidious/issues/2567
+##
+## Subtitle experience may differ slightly on Invidious.
+##
+## Accepted values: true, false
+## Default: false
+##
+# use_innertube_for_captions: false
+
+
 # -----------------------------
 #  Logging
 # -----------------------------

src/invidious/config.cr

@@ -127,6 +127,9 @@ class Config
   # Pool size for HTTP requests to youtube.com and ytimg.com (each domain has a separate pool of `pool_size`)
   property pool_size : Int32 = 100
 
+  # Use Innertube's transcripts API instead of timedtext for closed captions
+  property use_innertube_for_captions : Bool = false
+
   # Saved cookies in "name1=value1; name2=value2..." format
   @[YAML::Field(converter: Preferences::StringToCookies)]
   property cookies : HTTP::Cookies = HTTP::Cookies.new

src/invidious/frontend/watch_page.cr

@@ -7,7 +7,7 @@ module Invidious::Frontend::WatchPage
     getter full_videos : Array(Hash(String, JSON::Any))
     getter video_streams : Array(Hash(String, JSON::Any))
     getter audio_streams : Array(Hash(String, JSON::Any))
-    getter captions : Array(Invidious::Videos::Caption)
+    getter captions : Array(Invidious::Videos::Captions::Metadata)
 
     def initialize(
       @full_videos,

src/invidious/routes/api/v1/videos.cr

@@ -87,6 +87,13 @@ module Invidious::Routes::API::V1::Videos
       caption = caption[0]
     end
 
+    if CONFIG.use_innertube_for_captions
+      params = Invidious::Videos::Transcript.generate_param(id, caption.language_code, caption.auto_generated)
+      initial_data = YoutubeAPI.get_transcript(params)
+
+      webvtt = Invidious::Videos::Transcript.convert_transcripts_to_vtt(initial_data, caption.language_code)
+    else
+      # Timedtext API handling
       url = URI.parse("#{caption.base_url}&tlang=#{tlang}").request_target
 
       # Auto-generated captions often have cues that aren't aligned properly with the video,
@@ -153,6 +160,7 @@ module Invidious::Routes::API::V1::Videos
             .gsub(/([0-9:.]{12} --> [0-9:.]{12}).+/, "\\1")
         end
       end
+    end
 
     if title = env.params.query["title"]?
       # https://blog.fastmail.com/2011/06/24/download-non-english-filenames/

src/invidious/videos.cr

@@ -24,7 +24,7 @@ struct Video
   property updated : Time
 
   @[DB::Field(ignore: true)]
-  @captions = [] of Invidious::Videos::Caption
+  @captions = [] of Invidious::Videos::Captions::Metadata
 
   @[DB::Field(ignore: true)]
   property adaptive_fmts : Array(Hash(String, JSON::Any))?
@@ -215,9 +215,9 @@ struct Video
     keywords.includes? "YouTube Red"
   end
 
-  def captions : Array(Invidious::Videos::Caption)
+  def captions : Array(Invidious::Videos::Captions::Metadata)
     if @captions.empty? && @info.has_key?("captions")
-      @captions = Invidious::Videos::Caption.from_yt_json(info["captions"])
+      @captions = Invidious::Videos::Captions::Metadata.from_yt_json(info["captions"])
     end
 
     return @captions

src/invidious/videos/caption.cr

@@ -1,21 +1,24 @@
 require "json"
 
 module Invidious::Videos
-  struct Caption
+  module Captions
+    struct Metadata
       property name : String
       property language_code : String
       property base_url : String
 
-    def initialize(@name, @language_code, @base_url)
+      property auto_generated : Bool
+
+      def initialize(@name, @language_code, @base_url, @auto_generated)
       end
 
       # Parse the JSON structure from Youtube
-    def self.from_yt_json(container : JSON::Any) : Array(Caption)
+      def self.from_yt_json(container : JSON::Any) : Array(Captions::Metadata)
         caption_tracks = container
           .dig?("playerCaptionsTracklistRenderer", "captionTracks")
           .try &.as_a
 
-      captions_list = [] of Caption
+        captions_list = [] of Captions::Metadata
         return captions_list if caption_tracks.nil?
 
         caption_tracks.each do |caption|
@@ -25,7 +28,9 @@ module Invidious::Videos
           language_code = caption["languageCode"].to_s
           base_url = caption["baseUrl"].to_s
 
-        captions_list << Caption.new(name, language_code, base_url)
+          auto_generated = (caption["kind"]? == "asr")
+
+          captions_list << Captions::Metadata.new(name, language_code, base_url, auto_generated)
         end
 
         return captions_list
@@ -96,6 +101,7 @@ module Invidious::Videos
         end
         return result
       end
+    end
 
     # List of all caption languages available on Youtube.
     LANGUAGES = {

src/invidious/videos/transcript.cr

@@ -0,0 +1,103 @@
+module Invidious::Videos
+  # Namespace for methods primarily relating to Transcripts
+  module Transcript
+    record TranscriptLine, start_ms : Time::Span, end_ms : Time::Span, line : String
+
+    def self.generate_param(video_id : String, language_code : String, auto_generated : Bool) : String
+      kind = auto_generated ? "asr" : ""
+
+      object = {
+        "1:0:string" => video_id,
+
+        "2:base64" => {
+          "1:string" => kind,
+          "2:string" => language_code,
+          "3:string" => "",
+        },
+
+        "3:varint" => 1_i64,
+        "5:string" => "engagement-panel-searchable-transcript-search-panel",
+        "6:varint" => 1_i64,
+        "7:varint" => 1_i64,
+        "8:varint" => 1_i64,
+      }
+
+      params = object.try { |i| Protodec::Any.cast_json(i) }
+        .try { |i| Protodec::Any.from_json(i) }
+        .try { |i| Base64.urlsafe_encode(i) }
+        .try { |i| URI.encode_www_form(i) }
+
+      return params
+    end
+
+    def self.convert_transcripts_to_vtt(initial_data : Hash(String, JSON::Any), target_language : String) : String
+      # Convert into array of TranscriptLine
+      lines = self.parse(initial_data)
+
+      # Taken from Invidious::Videos::Captions::Metadata.timedtext_to_vtt()
+      vtt = String.build do |vtt|
+        vtt << <<-END_VTT
+        WEBVTT
+        Kind: captions
+        Language: #{target_language}
+
+
+        END_VTT
+
+        vtt << "\n\n"
+
+        lines.each do |line|
+          start_time = line.start_ms
+          end_time = line.end_ms
+
+          # start_time
+          vtt << start_time.hours.to_s.rjust(2, '0')
+          vtt << ':' << start_time.minutes.to_s.rjust(2, '0')
+          vtt << ':' << start_time.seconds.to_s.rjust(2, '0')
+          vtt << '.' << start_time.milliseconds.to_s.rjust(3, '0')
+
+          vtt << " --> "
+
+          # end_time
+          vtt << end_time.hours.to_s.rjust(2, '0')
+          vtt << ':' << end_time.minutes.to_s.rjust(2, '0')
+          vtt << ':' << end_time.seconds.to_s.rjust(2, '0')
+          vtt << '.' << end_time.milliseconds.to_s.rjust(3, '0')
+
+          vtt << "\n"
+          vtt << line.line
+
+          vtt << "\n"
+          vtt << "\n"
+        end
+      end
+
+      return vtt
+    end
+
+    private def self.parse(initial_data : Hash(String, JSON::Any))
+      body = initial_data.dig("actions", 0, "updateEngagementPanelAction", "content", "transcriptRenderer",
+        "content", "transcriptSearchPanelRenderer", "body", "transcriptSegmentListRenderer",
+        "initialSegments").as_a
+
+      lines = [] of TranscriptLine
+      body.each do |line|
+        # Transcript section headers. They are not apart of the captions and as such we can safely skip them.
+        if line.as_h.has_key?("transcriptSectionHeaderRenderer")
+          next
+        end
+
+        line = line["transcriptSegmentRenderer"]
+
+        start_ms = line["startMs"].as_s.to_i.millisecond
+        end_ms = line["endMs"].as_s.to_i.millisecond
+
+        text = extract_text(line["snippet"]) || ""
+
+        lines << TranscriptLine.new(start_ms, end_ms, text)
+      end
+
+      return lines
+    end
+  end
+end

src/invidious/views/user/preferences.ecr

@@ -89,7 +89,7 @@
                 <label for="captions[0]"><%= translate(locale, "preferences_captions_label") %></label>
                 <% preferences.captions.each_with_index do |caption, index| %>
                     <select class="pure-u-1-6" name="captions[<%= index %>]" id="captions[<%= index %>]">
-                        <% Invidious::Videos::Caption::LANGUAGES.each do |option| %>
+                        <% Invidious::Videos::Captions::LANGUAGES.each do |option| %>
                             <option value="<%= option %>" <% if preferences.captions[index] == option %> selected <% end %>><%= translate(locale, option.blank? ? "none" : option) %></option>
                         <% end %>
                     </select>

src/invidious/yt_backend/youtube_api.cr

@@ -557,6 +557,30 @@ module YoutubeAPI
     return self._post_json("/youtubei/v1/search", data, client_config)
   end
 
+  ####################################################################
+  # get_transcript(params, client_config?)
+  #
+  # Requests the youtubei/v1/get_transcript endpoint with the required headers
+  # and POST data in order to get a JSON reply.
+  #
+  # The requested data is a specially encoded protobuf string that denotes the specific language requested.
+  #
+  # An optional ClientConfig parameter can be passed, too (see
+  # `struct ClientConfig` above for more details).
+  #
+
+  def get_transcript(
+    params : String,
+    client_config : ClientConfig | Nil = nil
+  ) : Hash(String, JSON::Any)
+    data = {
+      "context" => self.make_context(client_config),
+      "params"  => params,
+    }
+
+    return self._post_json("/youtubei/v1/get_transcript", data, client_config)
+  end
+
   ####################################################################
   # _post_json(endpoint, data, client_config?)
   #