Storyboards: Various fixes and code cleaning (#4153)

Closes issue 3441

src/invidious/jsonify/api_v1/video_json.cr

@@ -277,17 +277,17 @@ module Invidious::JSONify::APIv1
 
   def storyboards(json, id, storyboards)
     json.array do
-      storyboards.each do |storyboard|
+      storyboards.each do |sb|
         json.object do
-          json.field "url", "/api/v1/storyboards/#{id}?width=#{storyboard[:width]}&height=#{storyboard[:height]}"
+          json.field "url", "/api/v1/storyboards/#{id}?width=#{sb.width}&height=#{sb.height}"
-          json.field "templateUrl", storyboard[:url]
+          json.field "templateUrl", sb.url.to_s
-          json.field "width", storyboard[:width]
+          json.field "width", sb.width
-          json.field "height", storyboard[:height]
+          json.field "height", sb.height
-          json.field "count", storyboard[:count]
+          json.field "count", sb.count
-          json.field "interval", storyboard[:interval]
+          json.field "interval", sb.interval
-          json.field "storyboardWidth", storyboard[:storyboard_width]
+          json.field "storyboardWidth", sb.columns
-          json.field "storyboardHeight", storyboard[:storyboard_height]
+          json.field "storyboardHeight", sb.rows
-          json.field "storyboardCount", storyboard[:storyboard_count]
+          json.field "storyboardCount", sb.images_count
         end
       end
     end

src/invidious/routes/api/v1/videos.cr

@@ -1,3 +1,5 @@
+require "html"
+
 module Invidious::Routes::API::V1::Videos
   def self.videos(env)
     locale = env.get("preferences").as(Preferences).locale
@@ -187,15 +189,14 @@ module Invidious::Routes::API::V1::Videos
       haltf env, 500
     end
 
-    storyboards = video.storyboards
+    width = env.params.query["width"]?.try &.to_i
-    width = env.params.query["width"]?
+    height = env.params.query["height"]?.try &.to_i
-    height = env.params.query["height"]?
 
     if !width && !height
       response = JSON.build do |json|
         json.object do
           json.field "storyboards" do
-            Invidious::JSONify::APIv1.storyboards(json, id, storyboards)
+            Invidious::JSONify::APIv1.storyboards(json, id, video.storyboards)
           end
         end
       end
@@ -205,35 +206,48 @@ module Invidious::Routes::API::V1::Videos
 
     env.response.content_type = "text/vtt"
 
-    storyboard = storyboards.select { |sb| width == "#{sb[:width]}" || height == "#{sb[:height]}" }
+    # Select a storyboard matching the user's provided width/height
+    storyboard = video.storyboards.select { |x| x.width == width || x.height == height }
+    haltf env, 404 if storyboard.empty?
 
-    if storyboard.empty?
+    # Alias variable, to make the code below esaier to read
-      haltf env, 404
+    sb = storyboard[0]
-    else
-      storyboard = storyboard[0]
-    end
 
-    WebVTT.build do |vtt|
+    # Some base URL segments that we'll use to craft the final URLs
-      start_time = 0.milliseconds
+    work_url = sb.proxied_url.dup
-      end_time = storyboard[:interval].milliseconds
+    template_path = sb.proxied_url.path
 
-      storyboard[:storyboard_count].times do |i|
+    # Initialize cue timing variables
-        url = storyboard[:url]
+    # NOTE: videojs-vtt-thumbnails gets lost when the cue times don't overlap
-        authority = /(i\d?).ytimg.com/.match!(url)[1]?
+    # (i.e: if cue[n] end time is 1:06:25.000, cue[n+1] start time should be 1:06:25.000)
-        url = url.gsub("$M", i).gsub(%r(https://i\d?.ytimg.com/sb/), "")
+    time_delta = sb.interval.milliseconds
-        url = "#{HOST_URL}/sb/#{authority}/#{url}"
+    start_time = 0.milliseconds
+    end_time = time_delta
 
-        storyboard[:storyboard_height].times do |j|
+    # Build a VTT file for VideoJS-vtt plugin
-          storyboard[:storyboard_width].times do |k|
+    vtt_file = WebVTT.build do |vtt|
-            current_cue_url = "#{url}#xywh=#{storyboard[:width] * k},#{storyboard[:height] * j},#{storyboard[:width] - 2},#{storyboard[:height]}"
+      sb.images_count.times do |i|
-            vtt.cue(start_time, end_time, current_cue_url)
+        # Replace the variable component part of the path
+        work_url.path = template_path.sub("$M", i)
 
-            start_time += storyboard[:interval].milliseconds
+        sb.rows.times do |j|
-            end_time += storyboard[:interval].milliseconds
+          sb.columns.times do |k|
+            # The URL fragment represents the offset of the thumbnail inside the storyboard image
+            work_url.fragment = "xywh=#{sb.width * k},#{sb.height * j},#{sb.width - 2},#{sb.height}"
+
+            vtt.cue(start_time, end_time, work_url.to_s)
+
+            start_time += time_delta
+            end_time += time_delta
           end
         end
       end
     end
+
+    # videojs-vtt-thumbnails is not compliant to the VTT specification, it
+    # doesn't unescape the HTML entities, so we have to do it here:
+    # TODO: remove this when we migrate to VideoJS 8
+    return HTML.unescape(vtt_file)
   end
 
   def self.annotations(env)

src/invidious/videos.cr

@@ -177,65 +177,8 @@ struct Video
   # Misc. methods
 
   def storyboards
-    storyboards = info.dig?("storyboards", "playerStoryboardSpecRenderer", "spec")
+    container = info.dig?("storyboards") || JSON::Any.new("{}")
-      .try &.as_s.split("|")
+    return IV::Videos::Storyboard.from_yt_json(container, self.length_seconds)
-
-    if !storyboards
-      if storyboard = info.dig?("storyboards", "playerLiveStoryboardSpecRenderer", "spec").try &.as_s
-        return [{
-          url:               storyboard.split("#")[0],
-          width:             106,
-          height:            60,
-          count:             -1,
-          interval:          5000,
-          storyboard_width:  3,
-          storyboard_height: 3,
-          storyboard_count:  -1,
-        }]
-      end
-    end
-
-    items = [] of NamedTuple(
-      url: String,
-      width: Int32,
-      height: Int32,
-      count: Int32,
-      interval: Int32,
-      storyboard_width: Int32,
-      storyboard_height: Int32,
-      storyboard_count: Int32)
-
-    return items if !storyboards
-
-    url = URI.parse(storyboards.shift)
-    params = HTTP::Params.parse(url.query || "")
-
-    storyboards.each_with_index do |sb, i|
-      width, height, count, storyboard_width, storyboard_height, interval, _, sigh = sb.split("#")
-      params["sigh"] = sigh
-      url.query = params.to_s
-
-      width = width.to_i
-      height = height.to_i
-      count = count.to_i
-      interval = interval.to_i
-      storyboard_width = storyboard_width.to_i
-      storyboard_height = storyboard_height.to_i
-      storyboard_count = (count / (storyboard_width * storyboard_height)).ceil.to_i
-
-      items << {
-        url:               url.to_s.sub("$L", i).sub("$N", "M$M"),
-        width:             width,
-        height:            height,
-        count:             count,
-        interval:          interval,
-        storyboard_width:  storyboard_width,
-        storyboard_height: storyboard_height,
-        storyboard_count:  storyboard_count,
-      }
-    end
-
-    items
   end
 
   def paid

src/invidious/videos/storyboard.cr

@@ -0,0 +1,122 @@
+require "uri"
+require "http/params"
+
+module Invidious::Videos
+  struct Storyboard
+    # Template URL
+    getter url : URI
+    getter proxied_url : URI
+
+    # Thumbnail parameters
+    getter width : Int32
+    getter height : Int32
+    getter count : Int32
+    getter interval : Int32
+
+    # Image (storyboard) parameters
+    getter rows : Int32
+    getter columns : Int32
+    getter images_count : Int32
+
+    def initialize(
+      *, @url, @width, @height, @count, @interval,
+      @rows, @columns, @images_count
+    )
+      authority = /(i\d?).ytimg.com/.match!(@url.host.not_nil!)[1]?
+
+      @proxied_url = URI.parse(HOST_URL)
+      @proxied_url.path = "/sb/#{authority}/#{@url.path.lchop("/sb/")}"
+      @proxied_url.query = @url.query
+    end
+
+    # Parse the JSON structure from Youtube
+    def self.from_yt_json(container : JSON::Any, length_seconds : Int32) : Array(Storyboard)
+      # Livestream storyboards are a bit different
+      # TODO: document exactly how
+      if storyboard = container.dig?("playerLiveStoryboardSpecRenderer", "spec").try &.as_s
+        return [Storyboard.new(
+          url: URI.parse(storyboard.split("#")[0]),
+          width: 106,
+          height: 60,
+          count: -1,
+          interval: 5000,
+          rows: 3,
+          columns: 3,
+          images_count: -1
+        )]
+      end
+
+      # Split the storyboard string into chunks
+      #
+      # General format (whitespaces added for legibility):
+      #   https://i.ytimg.com/sb/<video_id>/storyboard3_L$L/$N.jpg?sqp=<sig0>
+      #   |  48 #  27 #  100 #  10 #  10 #      0 #  default #  rs$<sig1>
+      #   |  80 #  45 #   95 #  10 #  10 #  10000 #      M$M #  rs$<sig2>
+      #   | 160 #  90 #   95 #   5 #   5 #  10000 #      M$M #  rs$<sig3>
+      #
+      storyboards = container.dig?("playerStoryboardSpecRenderer", "spec")
+        .try &.as_s.split("|")
+
+      return [] of Storyboard if !storyboards
+
+      # The base URL is the first chunk
+      base_url = URI.parse(storyboards.shift)
+
+      return storyboards.map_with_index do |sb, i|
+        # Separate the different storyboard parameters:
+        # width/height: respective dimensions, in pixels, of a single thumbnail
+        # count: how many thumbnails are displayed across the full video
+        # columns/rows: maximum amount of thumbnails that can be stuffed in a
+        #   single image, horizontally and vertically.
+        # interval: interval between two thumbnails, in milliseconds
+        # name: storyboard filename. Usually "M$M" or "default"
+        # sigh: URL cryptographic signature
+        width, height, count, columns, rows, interval, name, sigh = sb.split("#")
+
+        width = width.to_i
+        height = height.to_i
+        count = count.to_i
+        interval = interval.to_i
+        columns = columns.to_i
+        rows = rows.to_i
+
+        # Copy base URL object, so that we can modify it
+        url = base_url.dup
+
+        # Add the signature to the URL
+        params = url.query_params
+        params["sigh"] = sigh
+        url.query_params = params
+
+        # Replace the template parts with what we have
+        url.path = url.path.sub("$L", i).sub("$N", name)
+
+        # This value represents the maximum amount of thumbnails that can fit
+        # in a single image. The last image (or the only one for short videos)
+        # will contain less thumbnails than that.
+        thumbnails_per_image = columns * rows
+
+        # This value represents the total amount of storyboards required to
+        # hold all of the thumbnails. It can't be less than 1.
+        images_count = (count / thumbnails_per_image).ceil.to_i
+
+        # Compute the interval when needed (in general, that's only required
+        # for the first "default" storyboard).
+        if interval == 0
+          interval = ((length_seconds / count) * 1_000).to_i
+        end
+
+        Storyboard.new(
+          url: url,
+          width: width,
+          height: height,
+          count: count,
+          interval: interval,
+          rows: rows,
+          columns: columns,
+          images_count: images_count,
+        )
+      end
+    end
+  end
+end