Add option to disable easy to abuse API endpoints (#5630)

* Add option to disable easy to abuse API endpoints

The API endpoints that will be disabled when this option are:
 - /api/v1/videos
 - /api/v1/clips
 - /api/v1/transcripts

There is still API endponts that need some sort of validation or
connection/proxying to Invidious companion like
`/api/v1/captions` and `/api/v1/storyboards` since they also do a video
request to Invidious companion. I'm not sure if the `/next` API endpoint
could be used to gather that type of information, if so, that would be
better.

Closes #5599

* Rename configuration variable, add additional comment for the option

.github/workflows/ci.yml

@@ -65,7 +65,7 @@ jobs:
           crystal: ${{ matrix.crystal }}
 
       - name: Cache Shards
-        uses: actions/cache@v5
+        uses: actions/cache@v6
         with:
           path: |
             ./lib
@@ -137,7 +137,7 @@ jobs:
           crystal: latest
 
       - name: Cache Shards
-        uses: actions/cache@v5
+        uses: actions/cache@v6
         with:
           path: |
             ./lib

config/config.example.yml

@@ -205,7 +205,6 @@ https_only: false
 #  path: /tmp/invidious.sock
 #  permissions: 777
 
-
 # -----------------------------
 #  Network (outbound)
 # -----------------------------
@@ -228,7 +227,6 @@ https_only: false
 ##
 #pool_size: 100
 
-
 ##
 ## Additional cookies to be sent when requesting the youtube API.
 ##
@@ -263,7 +261,6 @@ https_only: false
 #  host:
 #  port:
 
-
 ##
 ## Use Innertube's transcripts API instead of timedtext for closed captions
 ##
@@ -344,7 +341,6 @@ https_only: false
 ##
 #statistics_enabled: false
 
-
 # -----------------------------
 #  Users and accounts
 # -----------------------------
@@ -456,12 +452,25 @@ full_refresh: false
 ##
 feed_threads: 1
 
+##
+## Setting to disable easy to abuse API endpoints that can
+## be spammed and therefore blocking your Invidious instance.
+##
+## Useful for public instance maintainers.
+##
+## Notes: The following API endpoints will be disabled:
+##  - /api/v1/videos
+##  - /api/v1/clips
+##  - /api/v1/transcripts
+##
+## Accepted values: true, false
+## Default: false
+##
+disable_abusable_api: false
 
 jobs:
-
   ## Options for the database cleaning job
   clear_expired_items:
-
     ## Enable/Disable job
     ##
     ## Accepted values: true, false
@@ -471,7 +480,6 @@ jobs:
 
   ## Options for the channels updater job
   refresh_channels:
-
     ## Enable/Disable job
     ##
     ## Accepted values: true, false
@@ -481,7 +489,6 @@ jobs:
 
   ## Options for the RSS feeds updater job
   refresh_feeds:
-
     ## Enable/Disable job
     ##
     ## Accepted values: true, false
@@ -489,7 +496,6 @@ jobs:
     ##
     enable: true
 
-
 # -----------------------------
 #  Miscellaneous
 # -----------------------------
@@ -688,7 +694,6 @@ default_user_preferences:
   ##
   #captions: ["", "", ""]
 
-
   # -----------------------------
   #  Interface
   # -----------------------------
@@ -790,7 +795,6 @@ default_user_preferences:
   ##
   #related_videos: true
 
-
   # -----------------------------
   #  Video player behavior
   # -----------------------------
@@ -854,7 +858,6 @@ default_user_preferences:
   ##
   #video_loop: false
 
-
   # -----------------------------
   #  Video playback settings
   # -----------------------------
@@ -966,7 +969,6 @@ default_user_preferences:
   ##
   #sort: published
 
-
   # -----------------------------
   #  Miscellaneous
   # -----------------------------

src/invidious.cr

@@ -217,6 +217,7 @@ end
 Kemal.config.powered_by_header = false
 add_handler FilteredCompressHandler.new
 add_handler APIHandler.new
+add_handler DisableAbusableAPIHandler.new
 add_handler AuthHandler.new
 add_handler DenyFrame.new
 

src/invidious/config.cr

@@ -183,6 +183,9 @@ class Config
   # Playlist length limit
   property playlist_length_limit : Int32 = 500
 
+  # Disable easy to abuse API endpoints
+  property disable_abusable_api : Bool = false
+
   def disabled?(option)
     case disabled = CONFIG.disable_proxy
     when Bool

src/invidious/database/playlists.cr

@@ -194,13 +194,13 @@ module Invidious::Database::PlaylistVideos
     PG_DB.exec(request, args: video_array)
   end
 
-  def delete(index)
+  def delete(index, plid : String)
     request = <<-SQL
       DELETE FROM playlist_videos *
-      WHERE index = $1
+      WHERE index = $1 AND plid = $2
     SQL
 
-    PG_DB.exec(request, index)
+    PG_DB.exec(request, index, plid)
   end
 
   def delete_by_playlist(plid : String)

src/invidious/helpers/handlers.cr

@@ -133,6 +133,26 @@ class APIHandler < Kemal::Handler
   end
 end
 
+class DisableAbusableAPIHandler < Kemal::Handler
+  {% for method in %w(GET HEAD) %}
+    # This endpoints make a video request to Invidious companion.
+    {% for endpoint in %w(videos clips transcripts) %}
+      only ["/api/v1/{{ endpoint.id }}/:id"], {{ method }}
+    {% end %}
+  {% end %}
+
+  def call(env)
+    return call_next env unless only_match?(env) && CONFIG.disable_abusable_api
+
+    env.response.content_type = "application/json"
+    env.response.status_code = 403
+    message = {"error" => "This API endpoint has been disabled by the administrator."}.to_json
+    env.response.print message
+    env.response.close
+    return
+  end
+end
+
 class DenyFrame < Kemal::Handler
   exclude ["/embed/*"]
 

src/invidious/jobs/refresh_channels_job.cr

@@ -36,6 +36,11 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
 
               LOGGER.trace("RefreshChannelsJob: #{id} fiber : Updating DB")
               Invidious::Database::Channels.update_author(id, channel.author)
+
+              if backoff > 2.minutes
+                backoff /= 2
+                LOGGER.debug("RefreshChannelsJob: #{id} fiber : decreasing backoff to #{backoff}s")
+              end
             rescue ex
               LOGGER.error("RefreshChannelsJob: #{id} : #{ex.message}")
               if ex.message == "Deleted or invalid channel"

src/invidious/routes/api/v1/authenticated.cr

@@ -364,7 +364,7 @@ module Invidious::Routes::API::V1::Authenticated
       return error_json(404, "Playlist does not contain index")
     end
 
-    Invidious::Database::PlaylistVideos.delete(index)
+    Invidious::Database::PlaylistVideos.delete(index, plid)
     Invidious::Database::Playlists.update_video_removed(plid, index)
 
     env.response.status_code = 204

src/invidious/routes/playlists.cr

@@ -357,8 +357,12 @@ module Invidious::Routes::Playlists
       Invidious::Database::PlaylistVideos.insert(playlist_video)
       Invidious::Database::Playlists.update_video_added(playlist_id, playlist_video.index)
     when "remove_video"
-      index = env.params.query["set_video_id"]
-      Invidious::Database::PlaylistVideos.delete(index)
+      index = env.params.query["set_video_id"].to_i64?
+      if index.nil? || !playlist.index.includes? index
+        return error_json(404, "Playlist does not contain index")
+      end
+
+      Invidious::Database::PlaylistVideos.delete(index, playlist_id)
       Invidious::Database::Playlists.update_video_removed(playlist_id, index)
     when "move_video_before"
       # TODO: Playlist stub

src/invidious/yt_backend/connection_pool.cr

@@ -106,7 +106,7 @@ end
 
 def add_yt_headers(request)
   request.headers.delete("User-Agent") if request.headers["User-Agent"] == "Crystal"
-  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36"
+  request.headers["User-Agent"] ||= "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.0.0 Safari/537.36"
 
   request.headers["Accept-Charset"] ||= "ISO-8859-1,utf-8;q=0.7,*;q=0.7"
   request.headers["Accept"] ||= "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"