From b5d2eb5c708174f728f4961ee665166979e640dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos?= Date: Mon, 16 Aug 2021 19:41:16 +0200 Subject: [PATCH] fetch with innertube api when video is unavailable (#2329) + rename some client type to better names + fix thirdParty hack --- src/invidious/helpers/youtube_api.cr | 25 +++++------- src/invidious/videos.cr | 60 ++++++---------------------- 2 files changed, 22 insertions(+), 63 deletions(-) diff --git a/src/invidious/helpers/youtube_api.cr b/src/invidious/helpers/youtube_api.cr index 4ed707f6..b3815f6a 100644 --- a/src/invidious/helpers/youtube_api.cr +++ b/src/invidious/helpers/youtube_api.cr @@ -8,12 +8,12 @@ module YoutubeAPI # Enumerate used to select one of the clients supported by the API enum ClientType Web - WebEmbed + WebEmbeddedPlayer WebMobile - WebAgeBypass + WebScreenEmbed Android - AndroidEmbed - AndroidAgeBypass + AndroidEmbeddedPlayer + AndroidScreenEmbed end # List of hard-coded values used by the different clients @@ -24,7 +24,7 @@ module YoutubeAPI api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", screen: "WATCH_FULL_SCREEN", }, - ClientType::WebEmbed => { + ClientType::WebEmbeddedPlayer => { name: "WEB_EMBEDDED_PLAYER", # 56 version: "1.20210721.1.0", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", @@ -36,7 +36,7 @@ module YoutubeAPI api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", screen: "", # None }, - ClientType::WebAgeBypass => { + ClientType::WebScreenEmbed => { name: "WEB", version: "2.20210721.00.00", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", @@ -48,13 +48,13 @@ module YoutubeAPI api_key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", screen: "", # ?? }, - ClientType::AndroidEmbed => { + ClientType::AndroidEmbeddedPlayer => { name: "ANDROID_EMBEDDED_PLAYER", # 55 version: "16.20", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", screen: "", # None? }, - ClientType::AndroidAgeBypass => { + ClientType::AndroidScreenEmbed => { name: "ANDROID", # 3 version: "16.20", api_key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", @@ -156,9 +156,6 @@ module YoutubeAPI "gl" => client_config.region || "US", # Can't be empty! "clientName" => client_config.name, "clientVersion" => client_config.version, - "thirdParty" => { - "embedUrl" => "", # Placeholder - }, }, } @@ -167,14 +164,10 @@ module YoutubeAPI client_context["client"]["clientScreen"] = client_config.screen end - # Replacing/removing the placeholder is easier than trying to - # merge two different Hash structures. if client_config.screen == "EMBED" - client_context["client"]["thirdParty"] = { + client_context["thirdParty"] = { "embedUrl" => "https://www.youtube.com/embed/dQw4w9WgXcQ", } - else - client_context["client"].delete("thirdParty") end return client_context diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 6a9c328e..27d85b92 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -819,10 +819,14 @@ def parse_related(r : JSON::Any) : JSON::Any? JSON::Any.new(rv) end -def extract_video_info(video_id : String, proxy_region : String? = nil) +def extract_video_info(video_id : String, proxy_region : String? = nil, context_screen : String? = nil) params = {} of String => JSON::Any client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) + if context_screen == "embed" + client_config.client_type = YoutubeAPI::ClientType::WebScreenEmbed + end + player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) if player_response["playabilityStatus"]?.try &.["status"]?.try &.as_s != "OK" @@ -844,7 +848,11 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) # maybe fix throttling issues (#2194).See for the explanation about the decrypted URLs: # https://github.com/TeamNewPipe/NewPipeExtractor/issues/562 if !params["reason"]? - client_config.client_type = YoutubeAPI::ClientType::Android + if context_screen == "embed" + client_config.client_type = YoutubeAPI::ClientType::AndroidScreenEmbed + else + client_config.client_type = YoutubeAPI::ClientType::Android + end stream_data = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) params["streamingData"] = stream_data["streamingData"]? || JSON::Any.new("") end @@ -972,52 +980,10 @@ def fetch_video(id, region) end end - # Try to pull streams from embed URL + # Try to fetch video info using an embedded client if info["reason"]? - required_parameters = { - "video_id" => id, - "eurl" => "https://youtube.googleapis.com/v/#{id}", - "html5" => "1", - "gl" => "US", - "hl" => "en", - } - if info["reason"].as_s.includes?("inappropriate") - # The html5, c and cver parameters are required in order to extract age-restricted videos - # See https://github.com/yt-dlp/yt-dlp/commit/4e6767b5f2e2523ebd3dd1240584ead53e8c8905 - required_parameters.merge!({ - "c" => "TVHTML5", - "cver" => "6.20180913", - }) - - # In order to actually extract video info without error, the `x-youtube-client-version` - # has to be set to the same version as `cver` above. - additional_headers = HTTP::Headers{"x-youtube-client-version" => "6.20180913"} - else - embed_page = YT_POOL.client &.get("/embed/#{id}").body - sts = embed_page.match(/"sts"\s*:\s*(?\d+)/).try &.["sts"]? || "" - required_parameters["sts"] = sts - additional_headers = HTTP::Headers{} of String => String - end - - embed_info = HTTP::Params.parse(YT_POOL.client &.get("/get_video_info?#{URI::Params.encode(required_parameters)}", - headers: additional_headers).body) - - if embed_info["player_response"]? - player_response = JSON.parse(embed_info["player_response"]) - {"captions", "microformat", "playabilityStatus", "streamingData", "videoDetails", "storyboards"}.each do |f| - info[f] = player_response[f] if player_response[f]? - end - end - - initial_data = JSON.parse(embed_info["watch_next_response"]) if embed_info["watch_next_response"]? - - info["relatedVideos"] = initial_data.try &.["playerOverlays"]?.try &.["playerOverlayRenderer"]? - .try &.["endScreen"]?.try &.["watchNextEndScreenRenderer"]?.try &.["results"]?.try &.as_a.compact_map { |r| - parse_related r - }.try { |a| JSON::Any.new(a) } || embed_info["rvs"]?.try &.split(",").map { |r| - r = HTTP::Params.parse(r).to_h - JSON::Any.new(Hash.zip(r.keys, r.values.map { |v| JSON::Any.new(v) })) - }.try { |a| JSON::Any.new(a) } || JSON::Any.new([] of JSON::Any) + embed_info = extract_video_info(video_id: id, context_screen: "embed") + info = embed_info if !embed_info["reason"]? end raise InfoException.new(info["reason"]?.try &.as_s || "") if !info["videoDetails"]?