Add wait time option and add handling for reddit_pool

src/helpers.cr

@@ -252,8 +252,8 @@ end
 
 def make_client(url, context)
   client = HTTP::Client.new(url, context)
-  client.read_timeout = 10.seconds
+  client.read_timeout = 30.seconds
-  client.connect_timeout = 10.seconds
+  client.connect_timeout = 30.seconds
   return client
 end
 

src/invidious.cr

@@ -21,7 +21,7 @@ require "xml"
 require "./helpers"
 
 threads = 10
-
+sleep_time = 1.0
 Kemal.config.extra_options do |parser|
   parser.banner = "Usage: invidious [arguments]"
   parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number|
@@ -32,6 +32,14 @@ Kemal.config.extra_options do |parser|
       exit
     end
   end
+  parser.on("-w TIME", "--wait=TIME", "Time to wait between server requests in seconds (default: 1 second)") do |number|
+    begin
+      sleep_time = number.to_f64
+    rescue ex
+      puts "TIME must be integer or float"
+      exit
+    end
+  end
 end
 
 Kemal::CLI.new
@@ -108,6 +116,8 @@ threads.times do
         end
       end
 
+      sleep sleep_time.seconds
+
       youtube_pool << client
     end
   end
@@ -118,8 +128,14 @@ threads.times do
     loop do
       client = get_client(reddit_pool)
 
+      begin
       client.get("/")
-      sleep 10.seconds
+      rescue ex
+        reddit_pool << make_client(URI.parse("https://api.reddit.com"), CONTEXT)
+        next
+      end
+
+      sleep sleep_time.seconds
 
       reddit_pool << client
     end