Add table cleaning job (#3294)

config/config.example.yml

@@ -304,10 +304,8 @@ https_only: false
 ## Number of threads to use when crawling channel videos (during
 ## subscriptions update).
 ##
-## Notes:
+## Notes: This setting is overridden if either "-c THREADS" or
-##  - Setting this to 0 will disable the channel videos crawl job.
+## "--channel-threads=THREADS" is passed on the command line.
-##  - This setting is overridden if "-c THREADS" or
-##    "--channel-threads=THREADS" are passed on the command line.
 ##
 ## Accepted values: a positive integer
 ## Default: 1
@@ -335,10 +333,8 @@ full_refresh: false
 ##
 ## Number of threads to use when updating RSS feeds.
 ##
-## Notes:
+## Notes: This setting is overridden if either "-f THREADS" or
-##  - Setting this to 0 will disable the channel videos crawl job.
+## "--feed-threads=THREADS" is passed on the command line.
-##  - This setting is overridden if "-f THREADS" or
-##    "--feed-threads=THREADS" are passed on the command line.
 ##
 ## Accepted values: a positive integer
 ## Default: 1
@@ -361,6 +357,39 @@ feed_threads: 1
 #decrypt_polling: false
 
 
+jobs:
+
+  ## Options for the database cleaning job
+  clear_expired_items:
+
+    ## Enable/Disable job
+    ##
+    ## Accepted values: true, false
+    ## Default: true
+    ##
+    enable: true
+
+  ## Options for the channels updater job
+  refresh_channels:
+
+    ## Enable/Disable job
+    ##
+    ## Accepted values: true, false
+    ## Default: true
+    ##
+    enable: true
+
+  ## Options for the RSS feeds updater job
+  refresh_feeds:
+
+    ## Enable/Disable job
+    ##
+    ## Accepted values: true, false
+    ## Default: true
+    ##
+    enable: true
+
+
 # -----------------------------
 #  Captcha API
 # -----------------------------

src/invidious.cr

@@ -172,6 +172,8 @@ end
 CONNECTION_CHANNEL = Channel({Bool, Channel(PQ::Notification)}).new(32)
 Invidious::Jobs.register Invidious::Jobs::NotificationJob.new(CONNECTION_CHANNEL, CONFIG.database_url)
 
+Invidious::Jobs.register Invidious::Jobs::ClearExpiredItemsJob.new
+
 Invidious::Jobs.start_all
 
 def popular_videos

src/invidious/config.cr

@@ -78,6 +78,10 @@ class Config
   property decrypt_polling : Bool = false
   # Used for crawling channels: threads should check all videos uploaded by a channel
   property full_refresh : Bool = false
+
+  # Jobs config structure. See jobs.cr and jobs/base_job.cr
+  property jobs = Invidious::Jobs::JobsConfig.new
+
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
   # HMAC signing key for CSRF tokens and verifying pubsub subscriptions

src/invidious/database/nonces.cr

@@ -4,7 +4,7 @@ module Invidious::Database::Nonces
   extend self
 
   # -------------------
-  #  Insert
+  #  Insert / Delete
   # -------------------
 
   def insert(nonce : String, expire : Time)
@@ -17,6 +17,15 @@ module Invidious::Database::Nonces
     PG_DB.exec(request, nonce, expire)
   end
 
+  def delete_expired
+    request = <<-SQL
+      DELETE FROM nonces *
+      WHERE expire < now()
+    SQL
+
+    PG_DB.exec(request)
+  end
+
   # -------------------
   #  Update
   # -------------------

src/invidious/database/videos.cr

@@ -22,6 +22,15 @@ module Invidious::Database::Videos
     PG_DB.exec(request, id)
   end
 
+  def delete_expired
+    request = <<-SQL
+      DELETE FROM videos *
+      WHERE updated < (now() - interval '6 hours')
+    SQL
+
+    PG_DB.exec(request)
+  end
+
   def update(video : Video)
     request = <<-SQL
       UPDATE videos

src/invidious/jobs.cr

@@ -1,12 +1,39 @@
 module Invidious::Jobs
   JOBS = [] of BaseJob
 
+  # Automatically generate a structure that wraps the various
+  # jobs' configs, so that the follwing YAML config can be used:
+  #
+  # jobs:
+  #   job_name:
+  #     enabled: true
+  #     some_property: "value"
+  #
+  macro finished
+    struct JobsConfig
+      include YAML::Serializable
+
+      {% for sc in BaseJob.subclasses %}
+        # Voodoo macro to transform `Some::Module::CustomJob` to `custom`
+        {% class_name = sc.id.split("::").last.id.gsub(/Job$/, "").underscore %}
+
+        getter {{ class_name }} = {{ sc.name }}::Config.new
+      {% end %}
+
+      def initialize
+      end
+    end
+  end
+
   def self.register(job : BaseJob)
     JOBS << job
   end
 
   def self.start_all
     JOBS.each do |job|
+      # Don't run the main rountine if the job is disabled by config
+      next if job.disabled?
+
       spawn { job.begin }
     end
   end

src/invidious/jobs/base_job.cr

@@ -1,3 +1,33 @@
 abstract class Invidious::Jobs::BaseJob
   abstract def begin
+
+  # When this base job class is inherited, make sure to define
+  # a basic "Config" structure, that contains the "enable" property,
+  # and to create the associated instance property.
+  #
+  macro inherited
+    macro finished
+      # This config structure can be expanded as required.
+      struct Config
+        include YAML::Serializable
+
+        property enable = true
+
+        def initialize
+        end
+      end
+
+      property cfg = Config.new
+
+      # Return true if job is enabled by config
+      protected def enabled? : Bool
+        return (@cfg.enable == true)
+      end
+
+      # Return true if job is disabled by config
+      protected def disabled? : Bool
+        return (@cfg.enable == false)
+      end
+    end
+  end
 end

src/invidious/jobs/clear_expired_items_job.cr

@@ -0,0 +1,27 @@
+class Invidious::Jobs::ClearExpiredItemsJob < Invidious::Jobs::BaseJob
+  # Remove items (videos, nonces, etc..) whose cache is outdated every hour.
+  # Removes the need for a cron job.
+  def begin
+    loop do
+      failed = false
+
+      LOGGER.info("jobs: running ClearExpiredItems job")
+
+      begin
+        Invidious::Database::Videos.delete_expired
+        Invidious::Database::Nonces.delete_expired
+      rescue DB::Error
+        failed = true
+      end
+
+      # Retry earlier than scheduled on DB error
+      if failed
+        LOGGER.info("jobs: ClearExpiredItems failed. Retrying in 10 minutes.")
+        sleep 10.minutes
+      else
+        LOGGER.info("jobs: ClearExpiredItems done.")
+        sleep 1.hour
+      end
+    end
+  end
+end