From fada57a307d66d696d9286fc943c579a3fd22de6 Mon Sep 17 00:00:00 2001
From: Omar Roth <omarroth@hotmail.com>
Date: Mon, 13 Aug 2018 09:17:28 -0500
Subject: [PATCH] Add geo-bypass for video info
---
src/invidious/helpers/proxy.cr | 143 +++++++++++++++++++++++++++++++++
src/invidious/videos.cr | 46 +++++++++++
src/invidious/views/watch.ecr | 9 +++
3 files changed, 198 insertions(+)
create mode 100644 src/invidious/helpers/proxy.cr
diff --git a/src/invidious/helpers/proxy.cr b/src/invidious/helpers/proxy.cr
new file mode 100644
index 00000000..3e94042d
--- /dev/null
+++ b/src/invidious/helpers/proxy.cr
@@ -0,0 +1,143 @@
+# See https://github.com/crystal-lang/crystal/issues/2963
+class HTTPProxy
+ getter proxy_host : String
+ getter proxy_port : Int32
+ getter options : Hash(Symbol, String)
+ getter tls : OpenSSL::SSL::Context::Client?
+
+ def initialize(@proxy_host, @proxy_port = 80, @options = {} of Symbol => String)
+ end
+
+ def open(host, port, tls = nil, connection_options = {} of Symbol => Float64 | Nil)
+ dns_timeout = connection_options.fetch(:dns_timeout, nil)
+ connect_timeout = connection_options.fetch(:connect_timeout, nil)
+ read_timeout = connection_options.fetch(:read_timeout, nil)
+
+ socket = TCPSocket.new @proxy_host, @proxy_port, dns_timeout, connect_timeout
+ socket.read_timeout = read_timeout if read_timeout
+ socket.sync = true
+
+ socket << "CONNECT #{host}:#{port} HTTP/1.1\r\n"
+
+ if options[:user]?
+ credentials = Base64.strict_encode("#{options[:user]}:#{options[:password]}")
+ credentials = "#{credentials}\n".gsub(/\s/, "")
+ socket << "Proxy-Authorization: Basic #{credentials}\r\n"
+ end
+
+ socket << "\r\n"
+
+ resp = parse_response(socket)
+
+ if resp[:code]? == 200
+ {% if !flag?(:without_openssl) %}
+ if tls
+ tls_socket = OpenSSL::SSL::Socket::Client.new(socket, context: tls, sync_close: true, hostname: host)
+ socket = tls_socket
+ end
+ {% end %}
+
+ return socket
+ else
+ socket.close
+ raise IO::Error.new(resp.inspect)
+ end
+ end
+
+ private def parse_response(socket)
+ resp = {} of Symbol => Int32 | String | Hash(String, String)
+
+ begin
+ version, code, reason = socket.gets.as(String).chomp.split(/ /, 3)
+
+ headers = {} of String => String
+
+ while (line = socket.gets.as(String)) && (line.chomp != "")
+ name, value = line.split(/:/, 2)
+ headers[name.strip] = value.strip
+ end
+
+ resp[:version] = version
+ resp[:code] = code.to_i
+ resp[:reason] = reason
+ resp[:headers] = headers
+ rescue
+ end
+
+ return resp
+ end
+end
+
+class HTTPClient < HTTP::Client
+ def set_proxy(proxy : HTTPProxy)
+ begin
+ @socket = proxy.open(host: @host, port: @port, tls: @tls, connection_options: proxy_connection_options)
+ rescue IO::Error
+ @socket = nil
+ end
+ end
+
+ def proxy_connection_options
+ opts = {} of Symbol => Float64 | Nil
+
+ opts[:dns_timeout] = @dns_timeout
+ opts[:connect_timeout] = @connect_timeout
+ opts[:read_timeout] = @read_timeout
+
+ return opts
+ end
+end
+
+def get_proxies(country_code = "US")
+ client = HTTP::Client.new(URI.parse("http://spys.one"))
+ client.read_timeout = 10.seconds
+ client.connect_timeout = 10.seconds
+
+ headers = HTTP::Headers.new
+ headers["User-Agent"] = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36"
+ headers["Accept"] = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8"
+ headers["Accept-Language"] = "Accept-Language: en-US,en;q=0.9"
+ headers["Host"] = "spys.one"
+ headers["Content-Type"] = "application/x-www-form-urlencoded"
+ body = {
+ "xpp" => "2",
+ "xf1" => "0",
+ "xf2" => "2",
+ "xf4" => "1",
+ "xf5" => "1",
+ }
+ response = client.post("/free-proxy-list/#{country_code}/", headers, form: body)
+ response = XML.parse_html(response.body)
+
+ proxies = [] of {ip: String, port: Int32, score: Float64}
+ response = response.xpath_nodes(%q(//table))[1]
+ response.xpath_nodes(%q(.//tr)).each do |node|
+ if !node["onmouseover"]?
+ next
+ end
+
+ ip = node.xpath_node(%q(.//td[1]/font[2])).to_s.match(/<font class="spy14">(?<address>[^<]+)</).not_nil!["address"]
+ port = 3128
+
+ latency = node.xpath_node(%q(.//td[6])).not_nil!.content.to_f
+ speed = node.xpath_node(%q(.//td[7]/font/table)).not_nil!["width"].to_f
+ uptime = node.xpath_node(%q(.//td[8]/font/acronym)).not_nil!
+
+ # Skip proxies that are down
+ if uptime["title"].ends_with? "?"
+ next
+ end
+
+ if md = uptime.content.match(/^\d+/)
+ uptime = md[0].to_f
+ else
+ next
+ end
+
+ score = (uptime*4 + speed*2 + latency)/7
+
+ proxies << {ip: ip, port: port, score: score}
+ end
+
+ return proxies
+end
diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr
index ed2c3d43..74a1b2c4 100644
--- a/src/invidious/videos.cr
+++ b/src/invidious/videos.cr
@@ -108,6 +108,9 @@ CAPTION_LANGUAGES = {
"Zulu",
}
+REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"}
+BYPASS_REGIONS = {"CA", "DE", "FR", "JP", "RU", "UK"}
+
VIDEO_THUMBNAILS = {
{name: "default", url: "default", height: 90, width: 120},
{name: "medium", url: "mqdefault", height: 180, width: 320},
@@ -409,6 +412,49 @@ def fetch_video(id)
html = html_channel.receive
info = info_channel.receive
+ if info["reason"]? && info["reason"].includes? "your country"
+ bypass_channel = Channel({HTTP::Params | Nil, XML::Node | Nil}).new
+
+ BYPASS_REGIONS.each do |country_code|
+ spawn do
+ begin
+ proxies = get_proxies(country_code)
+
+ # Try not to overload single proxy
+ proxy = proxies[0, 5].sample(1)[0]
+ proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
+
+ client = HTTPClient.new(URI.parse("https://www.youtube.com"))
+ client.read_timeout = 10.seconds
+ client.connect_timeout = 10.seconds
+ client.set_proxy(proxy)
+
+ proxy_info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1")
+ proxy_info = HTTP::Params.parse(proxy_info.body)
+
+ if !proxy_info["reason"]?
+ proxy_html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1")
+ proxy_html = XML.parse_html(proxy_html.body)
+
+ bypass_channel.send({proxy_info, proxy_html})
+ else
+ bypass_channel.send({nil, nil})
+ end
+ rescue ex
+ bypass_channel.send({nil, nil})
+ end
+ end
+ end
+
+ BYPASS_REGIONS.size.times do
+ response = bypass_channel.receive
+ if response[0] || response[1]
+ info = response[0].not_nil!
+ html = response[1].not_nil!
+ end
+ end
+ end
+
if info["reason"]?
raise info["reason"]
end
diff --git a/src/invidious/views/watch.ecr b/src/invidious/views/watch.ecr
index 822c87e8..ff27feb0 100644
--- a/src/invidious/views/watch.ecr
+++ b/src/invidious/views/watch.ecr
@@ -199,6 +199,15 @@ get_youtube_comments();
<p id="Wilson">Wilson Score : <%= video.wilson_score.round(4) %></p>
<p id="Rating">Rating : <%= rating.round(4) %> / 5</p>
<p id="Engagement">Engagement : <%= engagement.round(2) %>%</p>
+ <% if video.allowed_regions.size != REGIONS.size %>
+ <p id="AllowedRegions">
+ <% if video.allowed_regions.size < REGIONS.size / 2 %>
+ Whitelisted regions: <%= video.allowed_regions.join(", ") %>
+ <% else %>
+ Blacklisted regions: <%= (REGIONS.to_a - video.allowed_regions).join(", ") %>
+ <% end %>
+ </p>
+ <% end %>
<% if ad_slots %>
<p id="Slots">Ad Slots : <%= ad_slots %></p>
<% end %>