ProjectSegfault/invidious-experimenting
4
0
Fork 0
forked from midou/invidious
Code Pull Requests Activity

Add HTTPClient pool

Browse Source
This commit is contained in:
Omar Roth
2019-10-25 12:58:16 -04:00
parent aba2c5b938
commit 6930570fa2
14 changed files with 115 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/invidious/helpers/helpers.cr
View File

@@ -234,6 +234,7 @@ struct Config
force_resolve: {type: Socket::Family, default: Socket::Family::UNSPEC, converter: FamilyConverter}, # Connect to YouTube over 'ipv6', 'ipv4'. Will sometimes resolve fix issues with rate-limiting (see https://github.com/ytdl-org/youtube-dl/issues/21729)
port: {type: Int32, default: 3000}, # Port to listen for connections (overrided by command line argument)
host_binding: {type: String, default: "0.0.0.0"}, # Host to bind (overrided by command line argument)
pool_size: {type: Int32, default: 100},
})
end

4
src/invidious/helpers/proxy.cr
View File

@@ -77,6 +77,10 @@ class HTTPClient < HTTP::Client
end
end
def unset_proxy
@socket = nil
end
def proxy_connection_options
opts = {} of Symbol => Float64 | Nil

5
src/invidious/helpers/signatures.cr
View File

@@ -1,8 +1,7 @@
def fetch_decrypt_function(id = "CvFH_6DNRCY")
client = make_client(YT_URL)
document = client.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en&disable_polymer=1").body
url = document.match(/src="(?<url>\/yts\/jsbin\/player_ias-.{9}\/en_US\/base.js)"/).not_nil!["url"]
player = client.get(url).body
player = YT_POOL.client &.get(url).body
function_name = player.match(/^(?<name>[^=]+)=function\(a\){a=a\.split\(""\)/m).not_nil!["name"]
function_body = player.match(/^#{Regex.escape(function_name)}=function\(a\){(?<body>[^}]+)}/m).not_nil!["body"]

47
src/invidious/helpers/utils.cr
View File

@@ -1,3 +1,46 @@
require "pool/connection"
struct HTTPPool
property! url : URI
property! capacity : Int32
property! timeout : Float64
property pool : ConnectionPool(HTTPClient)
def initialize(url : URI, @capacity = 5, @timeout = 5.0)
@url = url
@pool = build_pool
end
def client(region = nil, &block)
pool.connection do |conn|
if region
PROXY_LIST[region]?.try &.sample(40).each do |proxy|
begin
proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
conn.set_proxy(proxy)
break
rescue ex
end
end
end
response = yield conn
conn.unset_proxy
response
end
end
private def build_pool
ConnectionPool(HTTPClient).new(capacity: capacity, timeout: timeout) do
client = HTTPClient.new(url)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
client.read_timeout = 5.seconds
client.connect_timeout = 5.seconds
client
end
end
end
# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
def ci_lower_bound(pos, n)
if n == 0
@@ -21,8 +64,8 @@ end
def make_client(url : URI, region = nil)
client = HTTPClient.new(url)
client.family = (url.host == "www.youtube.com") ? CONFIG.force_resolve : Socket::Family::UNSPEC
client.read_timeout = 15.seconds
client.connect_timeout = 15.seconds
client.read_timeout = 5.seconds
client.connect_timeout = 5.seconds
if region
PROXY_LIST[region]?.try &.sample(40).each do |proxy|