forked from midou/invidious
Merge pull request #1389 from vhuynh3000/decrypt_on_demand
add config to decrypt on demand instead of polling
This commit is contained in:
commit
2c24bf3222
@ -168,7 +168,11 @@ end
|
|||||||
Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config)
|
Invidious::Jobs.register Invidious::Jobs::RefreshChannelsJob.new(PG_DB, logger, config)
|
||||||
Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config)
|
Invidious::Jobs.register Invidious::Jobs::RefreshFeedsJob.new(PG_DB, logger, config)
|
||||||
Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY)
|
Invidious::Jobs.register Invidious::Jobs::SubscribeToFeedsJob.new(PG_DB, logger, config, HMAC_KEY)
|
||||||
Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new
|
|
||||||
|
DECRYPT_FUNCTION = DecryptFunction.new(CONFIG.decrypt_polling)
|
||||||
|
if config.decrypt_polling
|
||||||
|
Invidious::Jobs.register Invidious::Jobs::UpdateDecryptFunctionJob.new(logger)
|
||||||
|
end
|
||||||
|
|
||||||
if config.statistics_enabled
|
if config.statistics_enabled
|
||||||
Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE)
|
Invidious::Jobs.register Invidious::Jobs::StatisticsRefreshJob.new(PG_DB, config, SOFTWARE)
|
||||||
@ -191,8 +195,6 @@ def popular_videos
|
|||||||
Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
|
Invidious::Jobs::PullPopularVideosJob::POPULAR_VIDEOS.get
|
||||||
end
|
end
|
||||||
|
|
||||||
DECRYPT_FUNCTION = Invidious::Jobs::UpdateDecryptFunctionJob::DECRYPT_FUNCTION
|
|
||||||
|
|
||||||
before_all do |env|
|
before_all do |env|
|
||||||
preferences = begin
|
preferences = begin
|
||||||
Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}")
|
Preferences.from_json(env.request.cookies["PREFS"]?.try &.value || "{}")
|
||||||
|
@ -67,6 +67,7 @@ class Config
|
|||||||
property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
|
property channel_threads : Int32 # Number of threads to use for crawling videos from channels (for updating subscriptions)
|
||||||
property feed_threads : Int32 # Number of threads to use for updating feeds
|
property feed_threads : Int32 # Number of threads to use for updating feeds
|
||||||
property db : DBConfig # Database configuration
|
property db : DBConfig # Database configuration
|
||||||
|
property decrypt_polling : Bool = true # Use polling to keep decryption function up to date
|
||||||
property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
|
property full_refresh : Bool # Used for crawling channels: threads should check all videos uploaded by a channel
|
||||||
property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
|
property https_only : Bool? # Used to tell Invidious it is behind a proxy, so links to resources should be https://
|
||||||
property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
|
property hmac_key : String? # HMAC signing key for CSRF tokens and verifying pubsub subscriptions
|
||||||
|
@ -1,53 +1,73 @@
|
|||||||
alias SigProc = Proc(Array(String), Int32, Array(String))
|
alias SigProc = Proc(Array(String), Int32, Array(String))
|
||||||
|
|
||||||
def fetch_decrypt_function(id = "CvFH_6DNRCY")
|
struct DecryptFunction
|
||||||
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
|
@decrypt_function = [] of {SigProc, Int32}
|
||||||
url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
|
@decrypt_time = Time.monotonic
|
||||||
player = YT_POOL.client &.get(url).body
|
|
||||||
|
|
||||||
function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
|
def initialize(@use_polling = true)
|
||||||
function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"]
|
end
|
||||||
function_body = function_body.split(";")[1..-2]
|
|
||||||
|
|
||||||
var_name = function_body[0][0, 2]
|
def update_decrypt_function
|
||||||
var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"]
|
@decrypt_function = fetch_decrypt_function
|
||||||
|
end
|
||||||
|
|
||||||
operations = {} of String => SigProc
|
private def fetch_decrypt_function(id = "CvFH_6DNRCY")
|
||||||
var_body.split("},").each do |operation|
|
document = YT_POOL.client &.get("/watch?v=#{id}&gl=US&hl=en").body
|
||||||
op_name = operation.match(/^[^:]+/).not_nil![0]
|
url = document.match(/src="(?<url>\/s\/player\/[^\/]+\/player_ias[^\/]+\/en_US\/base.js)"/).not_nil!["url"]
|
||||||
op_body = operation.match(/\{[^}]+/).not_nil![0]
|
player = YT_POOL.client &.get(url).body
|
||||||
|
|
||||||
case op_body
|
function_name = player.match(/^(?<name>[^=]+)=function\(\w\){\w=\w\.split\(""\);[^\. ]+\.[^( ]+/m).not_nil!["name"]
|
||||||
when "{a.reverse()"
|
function_body = player.match(/^#{Regex.escape(function_name)}=function\(\w\){(?<body>[^}]+)}/m).not_nil!["body"]
|
||||||
operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
|
function_body = function_body.split(";")[1..-2]
|
||||||
when "{a.splice(0,b)"
|
|
||||||
operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
|
var_name = function_body[0][0, 2]
|
||||||
else
|
var_body = player.delete("\n").match(/var #{Regex.escape(var_name)}={(?<body>(.*?))};/).not_nil!["body"]
|
||||||
operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
|
|
||||||
|
operations = {} of String => SigProc
|
||||||
|
var_body.split("},").each do |operation|
|
||||||
|
op_name = operation.match(/^[^:]+/).not_nil![0]
|
||||||
|
op_body = operation.match(/\{[^}]+/).not_nil![0]
|
||||||
|
|
||||||
|
case op_body
|
||||||
|
when "{a.reverse()"
|
||||||
|
operations[op_name] = ->(a : Array(String), b : Int32) { a.reverse }
|
||||||
|
when "{a.splice(0,b)"
|
||||||
|
operations[op_name] = ->(a : Array(String), b : Int32) { a.delete_at(0..(b - 1)); a }
|
||||||
|
else
|
||||||
|
operations[op_name] = ->(a : Array(String), b : Int32) { c = a[0]; a[0] = a[b % a.size]; a[b % a.size] = c; a }
|
||||||
|
end
|
||||||
end
|
end
|
||||||
|
|
||||||
|
decrypt_function = [] of {SigProc, Int32}
|
||||||
|
function_body.each do |function|
|
||||||
|
function = function.lchop(var_name).delete("[].")
|
||||||
|
|
||||||
|
op_name = function.match(/[^\(]+/).not_nil![0]
|
||||||
|
value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i
|
||||||
|
|
||||||
|
decrypt_function << {operations[op_name], value}
|
||||||
|
end
|
||||||
|
|
||||||
|
return decrypt_function
|
||||||
end
|
end
|
||||||
|
|
||||||
decrypt_function = [] of {SigProc, Int32}
|
def decrypt_signature(fmt : Hash(String, JSON::Any))
|
||||||
function_body.each do |function|
|
return "" if !fmt["s"]? || !fmt["sp"]?
|
||||||
function = function.lchop(var_name).delete("[].")
|
|
||||||
|
|
||||||
op_name = function.match(/[^\(]+/).not_nil![0]
|
sp = fmt["sp"].as_s
|
||||||
value = function.match(/\(\w,(?<value>[\d]+)\)/).not_nil!["value"].to_i
|
sig = fmt["s"].as_s.split("")
|
||||||
|
if !@use_polling
|
||||||
|
now = Time.monotonic
|
||||||
|
if now - @decrypt_time > 60.seconds || @decrypt_function.size == 0
|
||||||
|
@decrypt_function = fetch_decrypt_function
|
||||||
|
@decrypt_time = Time.monotonic
|
||||||
|
end
|
||||||
|
end
|
||||||
|
|
||||||
decrypt_function << {operations[op_name], value}
|
@decrypt_function.each do |proc, value|
|
||||||
|
sig = proc.call(sig, value)
|
||||||
|
end
|
||||||
|
|
||||||
|
return "&#{sp}=#{sig.join("")}"
|
||||||
end
|
end
|
||||||
|
|
||||||
return decrypt_function
|
|
||||||
end
|
|
||||||
|
|
||||||
def decrypt_signature(fmt : Hash(String, JSON::Any))
|
|
||||||
return "" if !fmt["s"]? || !fmt["sp"]?
|
|
||||||
|
|
||||||
sp = fmt["sp"].as_s
|
|
||||||
sig = fmt["s"].as_s.split("")
|
|
||||||
DECRYPT_FUNCTION.each do |proc, value|
|
|
||||||
sig = proc.call(sig, value)
|
|
||||||
end
|
|
||||||
|
|
||||||
return "&#{sp}=#{sig.join("")}"
|
|
||||||
end
|
end
|
||||||
|
@ -1,15 +1,15 @@
|
|||||||
class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
|
class Invidious::Jobs::UpdateDecryptFunctionJob < Invidious::Jobs::BaseJob
|
||||||
DECRYPT_FUNCTION = [] of {SigProc, Int32}
|
private getter logger : Invidious::LogHandler
|
||||||
|
|
||||||
|
def initialize(@logger)
|
||||||
|
end
|
||||||
|
|
||||||
def begin
|
def begin
|
||||||
loop do
|
loop do
|
||||||
begin
|
begin
|
||||||
decrypt_function = fetch_decrypt_function
|
DECRYPT_FUNCTION.update_decrypt_function
|
||||||
DECRYPT_FUNCTION.clear
|
|
||||||
decrypt_function.each { |df| DECRYPT_FUNCTION << df }
|
|
||||||
rescue ex
|
rescue ex
|
||||||
# TODO: Log error
|
logger.error("UpdateDecryptFunctionJob : #{ex.message}")
|
||||||
next
|
|
||||||
ensure
|
ensure
|
||||||
sleep 1.minute
|
sleep 1.minute
|
||||||
Fiber.yield
|
Fiber.yield
|
||||||
|
@ -580,7 +580,7 @@ struct Video
|
|||||||
s.each do |k, v|
|
s.each do |k, v|
|
||||||
fmt[k] = JSON::Any.new(v)
|
fmt[k] = JSON::Any.new(v)
|
||||||
end
|
end
|
||||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
|
||||||
end
|
end
|
||||||
|
|
||||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
||||||
@ -599,7 +599,7 @@ struct Video
|
|||||||
s.each do |k, v|
|
s.each do |k, v|
|
||||||
fmt[k] = JSON::Any.new(v)
|
fmt[k] = JSON::Any.new(v)
|
||||||
end
|
end
|
||||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{decrypt_signature(fmt)}")
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}#{DECRYPT_FUNCTION.decrypt_signature(fmt)}")
|
||||||
end
|
end
|
||||||
|
|
||||||
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
fmt["url"] = JSON::Any.new("#{fmt["url"]}&host=#{URI.parse(fmt["url"].as_s).host}")
|
||||||
|
Loading…
Reference in New Issue
Block a user