From 323d27a5f6d4d997cd7ec44b27fad3c37325f9a4 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 3 Mar 2018 15:26:47 -0600 Subject: [PATCH] Add wait time option and add handling for reddit_pool --- src/helpers.cr | 4 ++-- src/invidious.cr | 20 ++++++++++++++++++-- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/src/helpers.cr b/src/helpers.cr index d0b6feece..eeda239aa 100644 --- a/src/helpers.cr +++ b/src/helpers.cr @@ -252,8 +252,8 @@ end def make_client(url, context) client = HTTP::Client.new(url, context) - client.read_timeout = 10.seconds - client.connect_timeout = 10.seconds + client.read_timeout = 30.seconds + client.connect_timeout = 30.seconds return client end diff --git a/src/invidious.cr b/src/invidious.cr index 7d870756e..a95e1fce4 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -21,7 +21,7 @@ require "xml" require "./helpers" threads = 10 - +sleep_time = 1.0 Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" parser.on("-t THREADS", "--threads=THREADS", "Number of threads for crawling (default: 10)") do |number| @@ -32,6 +32,14 @@ Kemal.config.extra_options do |parser| exit end end + parser.on("-w TIME", "--wait=TIME", "Time to wait between server requests in seconds (default: 1 second)") do |number| + begin + sleep_time = number.to_f64 + rescue ex + puts "TIME must be integer or float" + exit + end + end end Kemal::CLI.new @@ -108,6 +116,8 @@ threads.times do end end + sleep sleep_time.seconds + youtube_pool << client end end @@ -118,8 +128,14 @@ threads.times do loop do client = get_client(reddit_pool) + begin client.get("/") - sleep 10.seconds + rescue ex + reddit_pool << make_client(URI.parse("https://api.reddit.com"), CONTEXT) + next + end + + sleep sleep_time.seconds reddit_pool << client end