Merge pull request #2833 from matthewmcgarvey/get-channel-cleanup

Channel helpers cleanup

src/invidious/channels/channels.cr

@@ -114,8 +114,9 @@ class ChannelRedirect < Exception
   end
 end
 
-def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_threads = 10)
+def get_batch_channels(channels)
   finished_channel = Channel(String | Nil).new
+  max_threads = 10
 
   spawn do
     active_threads = 0
@@ -130,7 +131,7 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th
       active_threads += 1
       spawn do
         begin
-          get_channel(ucid, refresh, pull_all_videos)
+          get_channel(ucid)
           finished_channel.send(ucid)
         rescue ex
           finished_channel.send(nil)
@@ -151,23 +152,20 @@ def get_batch_channels(channels, refresh = false, pull_all_videos = true, max_th
   return final
 end
 
-def get_channel(id, refresh = true, pull_all_videos = true)
+def get_channel(id) : InvidiousChannel
-  if channel = Invidious::Database::Channels.select(id)
+  channel = Invidious::Database::Channels.select(id)
-    if refresh && Time.utc - channel.updated > 10.minutes
+
-      channel = fetch_channel(id, pull_all_videos: pull_all_videos)
+  if channel.nil? || (Time.utc - channel.updated) > 2.days
+    channel = fetch_channel(id, pull_all_videos: false)
     Invidious::Database::Channels.insert(channel, update_on_conflict: true)
   end
-  else
-    channel = fetch_channel(id, pull_all_videos: pull_all_videos)
-    Invidious::Database::Channels.insert(channel)
-  end
 
   return channel
 end
 
-def fetch_channel(ucid, pull_all_videos = true, locale = nil)
+def fetch_channel(ucid, pull_all_videos : Bool)
   LOGGER.debug("fetch_channel: #{ucid}")
-  LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}, locale = #{locale}")
+  LOGGER.trace("fetch_channel: #{ucid} : pull_all_videos = #{pull_all_videos}")
 
   LOGGER.trace("fetch_channel: #{ucid} : Downloading RSS feed")
   rss = YT_POOL.client &.get("/feeds/videos.xml?channel_id=#{ucid}").body

src/invidious/jobs/refresh_channels_job.cr

@@ -30,7 +30,7 @@ class Invidious::Jobs::RefreshChannelsJob < Invidious::Jobs::BaseJob
           spawn do
             begin
               LOGGER.trace("RefreshChannelsJob: #{id} fiber : Fetching channel")
-              channel = fetch_channel(id, CONFIG.full_refresh)
+              channel = fetch_channel(id, pull_all_videos: CONFIG.full_refresh)
 
               lim_fibers = max_fibers
 

src/invidious/routes/api/v1/authenticated.cr

@@ -92,7 +92,7 @@ module Invidious::Routes::API::V1::Authenticated
     ucid = env.params.url["ucid"]
 
     if !user.subscriptions.includes? ucid
-      get_channel(ucid, false, false)
+      get_channel(ucid)
       Invidious::Database::Users.subscribe_channel(user, ucid)
     end
 

src/invidious/routes/preferences.cr

@@ -327,7 +327,7 @@ module Invidious::Routes::PreferencesRoute
             user.subscriptions += body["subscriptions"].as_a.map(&.as_s)
             user.subscriptions.uniq!
 
-            user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+            user.subscriptions = get_batch_channels(user.subscriptions)
 
             Invidious::Database::Users.update_subscriptions(user)
           end
@@ -409,7 +409,7 @@ module Invidious::Routes::PreferencesRoute
           end
 
           user.subscriptions.uniq!
-          user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+          user.subscriptions = get_batch_channels(user.subscriptions)
 
           Invidious::Database::Users.update_subscriptions(user)
         when "import_freetube"
@@ -418,7 +418,7 @@ module Invidious::Routes::PreferencesRoute
           end
           user.subscriptions.uniq!
 
-          user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+          user.subscriptions = get_batch_channels(user.subscriptions)
 
           Invidious::Database::Users.update_subscriptions(user)
         when "import_newpipe_subscriptions"
@@ -437,7 +437,7 @@ module Invidious::Routes::PreferencesRoute
           end
           user.subscriptions.uniq!
 
-          user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+          user.subscriptions = get_batch_channels(user.subscriptions)
 
           Invidious::Database::Users.update_subscriptions(user)
         when "import_newpipe"
@@ -456,7 +456,7 @@ module Invidious::Routes::PreferencesRoute
                 user.subscriptions += db.query_all("SELECT url FROM subscriptions", as: String).map(&.lchop("https://www.youtube.com/channel/"))
                 user.subscriptions.uniq!
 
-                user.subscriptions = get_batch_channels(user.subscriptions, false, false)
+                user.subscriptions = get_batch_channels(user.subscriptions)
 
                 Invidious::Database::Users.update_subscriptions(user)
 

src/invidious/routes/subscriptions.cr

@@ -51,7 +51,7 @@ module Invidious::Routes::Subscriptions
     case action
     when "action_create_subscription_to_channel"
       if !user.subscriptions.includes? channel_id
-        get_channel(channel_id, false, false)
+        get_channel(channel_id)
         Invidious::Database::Users.subscribe_channel(user, channel_id)
       end
     when "action_remove_subscriptions"

src/invidious/users.cr

@@ -74,7 +74,7 @@ def fetch_user(sid, headers)
     end
   end
 
-  channels = get_batch_channels(channels, false, false)
+  channels = get_batch_channels(channels)
 
   email = feed.xpath_node(%q(//a[@class="yt-masthead-picker-header yt-masthead-picker-active-account"]))
   if email