From 92f78ff54135fd3f0723b7c434aab731cd41c1f7 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Sat, 28 Apr 2018 09:22:06 -0500 Subject: [PATCH] Remove client pool --- config/config.yml | 1 - src/helpers.cr | 15 ++--------- src/invidious.cr | 63 +++++++++++------------------------------------ 3 files changed, 17 insertions(+), 62 deletions(-) diff --git a/config/config.yml b/config/config.yml index 0dc93f86..3891c228 100644 --- a/config/config.yml +++ b/config/config.yml @@ -1,4 +1,3 @@ -pool_size: 20 channel_threads: 5 threads: 5 db: diff --git a/src/helpers.cr b/src/helpers.cr index 20477241..1506a3dd 100644 --- a/src/helpers.cr +++ b/src/helpers.cr @@ -15,7 +15,6 @@ end class Config YAML.mapping({ - pool_size: Int32, threads: Int32, channel_threads: Int32, db: NamedTuple( @@ -178,14 +177,6 @@ def elapsed_text(elapsed) "#{(millis * 1000).round(2)}µs" end -def get_client(pool) - while pool.empty? - sleep rand(0..10).milliseconds - end - - return pool.shift -end - def fetch_video(id, client) info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en").body html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}").body @@ -312,7 +303,7 @@ def decrypt_signature(a) return a.join("") end -def rank_videos(db, n, pool, filter) +def rank_videos(db, n, filter, url) top = [] of {Float64, String} db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs| @@ -339,15 +330,13 @@ def rank_videos(db, n, pool, filter) if language_list.size == n break else - client = get_client(pool) + client = make_client(url) begin video = get_video(id, client, db) rescue ex next end - pool << client - if video.language language = video.language else diff --git a/src/invidious.cr b/src/invidious.cr index a2afcd36..125659ff 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -25,20 +25,11 @@ require "./cookie_fix" CONFIG = Config.from_yaml(File.read("config/config.yml")) -pool_size = CONFIG.pool_size threads = CONFIG.threads channel_threads = CONFIG.channel_threads Kemal.config.extra_options do |parser| parser.banner = "Usage: invidious [arguments]" - parser.on("-z SIZE", "--youtube-pool=SIZE", "Number of clients in youtube pool (default: #{pool_size})") do |number| - begin - pool_size = number.to_i - rescue ex - puts "SIZE must be integer" - exit - end - end parser.on("-t THREADS", "--youtube-threads=THREADS", "Number of threads for crawling (default: #{threads})") do |number| begin threads = number.to_i @@ -73,25 +64,17 @@ YT_URL = URI.parse("https://www.youtube.com") REDDIT_URL = URI.parse("https://api.reddit.com") LOGIN_URL = URI.parse("https://accounts.google.com") -youtube_pool = Deque.new(pool_size) do - make_client(YT_URL) -end - -# Refresh youtube_pool by crawling YT threads.times do spawn do ids = Deque(String).new random = Random.new - client = get_client(youtube_pool) + client = make_client(YT_URL) search(random.base64(3), client) do |id| ids << id end - youtube_pool << client loop do - client = get_client(youtube_pool) - if ids.empty? search(random.base64(3), client) do |id| ids << id @@ -103,7 +86,7 @@ threads.times do video = get_video(id, client, PG_DB) rescue ex STDOUT << id << " : " << ex.message << "\n" - youtube_pool << make_client(YT_URL) + client = make_client(YT_URL) next ensure ids.delete(id) @@ -125,8 +108,6 @@ threads.times do end end end - - youtube_pool << client end end end @@ -139,16 +120,16 @@ channel_threads.times do |i| OFFSET (SELECT count(*)*$1/$2 FROM channels)" PG_DB.query(query, i, channel_threads) do |rs| rs.each do - client = get_client(youtube_pool) + client = make_client(YT_URL) + begin id = rs.read(String) channel = get_channel(id, client, PG_DB) rescue ex STDOUT << id << " : " << ex.message << "\n" - youtube_pool << make_client(YT_URL) + client = make_client(YT_URL) next end - youtube_pool << client end end end @@ -169,7 +150,7 @@ spawn do loop do begin - top = rank_videos(PG_DB, 40, youtube_pool, filter) + top = rank_videos(PG_DB, 40, filter, YT_URL) rescue ex next end @@ -183,13 +164,12 @@ spawn do videos = [] of Video top.each do |id| - client = get_client(youtube_pool) + client = make_client(YT_URL) begin videos << get_video(id, client, PG_DB) rescue ex next end - youtube_pool << client end top_videos = videos @@ -237,8 +217,6 @@ get "/watch" do |env| env.params.query.delete_all("listen") end - client = get_client(youtube_pool) - authorized = env.get? "authorized" if authorized sid = env.get("sid").as(String) @@ -248,13 +226,12 @@ get "/watch" do |env| subscriptions ||= [] of String + client = make_client(YT_URL) begin video = get_video(id, client, PG_DB) rescue ex error_message = ex.message next templated "error" - ensure - youtube_pool << client end fmt_stream = [] of HTTP::Params @@ -364,13 +341,11 @@ get "/search" do |env| page = env.params.query["page"]?.try &.to_i page ||= 1 - client = get_client(youtube_pool) + client = make_client(YT_URL) html = client.get("/results?q=#{URI.escape(query)}&page=#{page}&sp=EgIQAVAU").body html = XML.parse_html(html) - youtube_pool << client - videos = Array(Hash(String, String)).new html.xpath_nodes(%q(//ol[@class="item-section"]/li)).each do |item| @@ -499,9 +474,8 @@ post "/login" do |env| sid = login.cookies["SID"].value - client = get_client(youtube_pool) + client = make_client(YT_URL) user = get_user(sid, client, headers, PG_DB) - youtube_pool << client # We are now logged in @@ -552,13 +526,11 @@ get "/api/manifest/dash/id/:id" do |env| local = env.params.query["local"]?.try &.== "true" id = env.params.url["id"] - yt_client = get_client(youtube_pool) + client = make_client(YT_URL) begin - video = get_video(id, yt_client, PG_DB) + video = get_video(id, client, PG_DB) rescue ex halt env, status_code: 403 - ensure - youtube_pool << yt_client end adaptive_fmts = [] of HTTP::Params @@ -670,9 +642,8 @@ get "/feed/subscriptions" do |env| sid = env.get("sid").as(String) - client = get_client(youtube_pool) + client = make_client(YT_URL) user = get_user(sid, client, headers, PG_DB) - youtube_pool << client args = arg_array(user.subscriptions, 3) videos = PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid IN (#{args}) \ @@ -716,7 +687,7 @@ get "/modify_notifications" do |env| headers = HTTP::Headers.new headers["Cookie"] = env.request.headers["Cookie"] - client = get_client(youtube_pool) + client = make_client(YT_URL) subs = client.get("/subscription_manager?disable_polymer=1", headers) headers["Cookie"] += "; " + subs.cookies.add_request_headers(headers)["Cookie"] match = subs.body.match(/'XSRF_TOKEN': "(?[A-Za-z0-9\_\-\=]+)"/) @@ -737,8 +708,6 @@ get "/modify_notifications" do |env| client.post("/subscription_ajax?action_update_subscription_preferences=1", headers, HTTP::Params.encode(channel_req)).body end - - youtube_pool << client end env.redirect referer @@ -764,7 +733,7 @@ get "/subscription_ajax" do |env| headers = HTTP::Headers.new headers["Cookie"] = env.request.headers["Cookie"] - client = get_client(youtube_pool) + client = make_client(YT_URL) subs = client.get("/subscription_manager?disable_polymer=1", headers) headers["Cookie"] += "; " + subs.cookies.add_request_headers(headers)["Cookie"] @@ -795,8 +764,6 @@ get "/subscription_ajax" do |env| PG_DB.exec("UPDATE users SET subscriptions = array_remove(subscriptions,$1) WHERE id = $2", channel_id, sid) end end - - youtube_pool << client end env.redirect referer