From 4962c00ba8dda6496d3dbf51ffa2ddaf6718b876 Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Sat, 8 Jan 2022 12:21:36 -0600 Subject: [PATCH 1/2] Update to fetch related channels again --- src/invidious/channels/about.cr | 144 ++++++++++++------------ src/invidious/routes/api/v1/channels.cr | 28 ++--- 2 files changed, 86 insertions(+), 86 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index d93ee681d..48f2d4d29 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -1,33 +1,31 @@ # TODO: Refactor into either SearchChannel or InvidiousChannel -struct AboutChannel - include DB::Serializable +record AboutChannel, + ucid : String, + author : String, + auto_generated : Bool, + author_url : String, + author_thumbnail : String, + banner : String?, + description_html : String, + total_views : Int64, + sub_count : Int32, + joined : Time, + is_family_friendly : Bool, + allowed_regions : Array(String), + related_channels : RelatedChannels?, + tabs : Array(String) - property ucid : String - property author : String - property auto_generated : Bool - property author_url : String - property author_thumbnail : String - property banner : String? - property description_html : String - property total_views : Int64 - property sub_count : Int32 - property joined : Time - property is_family_friendly : Bool - property allowed_regions : Array(String) - property related_channels : Array(AboutRelatedChannel) - property tabs : Array(String) -end +record RelatedChannels, + browse_id : String, + params : String? -struct AboutRelatedChannel - include DB::Serializable +record AboutRelatedChannel, + ucid : String, + author : String, + author_url : String, + author_thumbnail : String - property ucid : String - property author : String - property author_url : String - property author_thumbnail : String -end - -def get_about_info(ucid, locale) +def get_about_info(ucid, locale) : AboutChannel begin # "EgVhYm91dA==" is the base64-encoded protobuf object {"2:string":"about"} initdata = YoutubeAPI.browse(browse_id: ucid, params: "EgVhYm91dA==") @@ -49,6 +47,7 @@ def get_about_info(ucid, locale) auto_generated = true end + related_channels = nil if auto_generated author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s @@ -63,8 +62,6 @@ def get_about_info(ucid, locale) is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) - - related_channels = [] of AboutRelatedChannel else author = initdata["metadata"]["channelMetadataRenderer"]["title"].as_s author_url = initdata["metadata"]["channelMetadataRenderer"]["channelUrl"].as_s @@ -86,37 +83,14 @@ def get_about_info(ucid, locale) is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) - related_channels = initdata["contents"]["twoColumnBrowseResultsRenderer"] - .["secondaryContents"]?.try &.["browseSecondaryContentsRenderer"]["contents"][0]? - .try &.["verticalChannelSectionRenderer"]?.try &.["items"]?.try &.as_a.map do |node| - renderer = node["miniChannelRenderer"]? - related_id = renderer.try &.["channelId"]?.try &.as_s? - related_id ||= "" - - related_title = renderer.try &.["title"]?.try &.["simpleText"]?.try &.as_s? - related_title ||= "" - - related_author_url = renderer.try &.["navigationEndpoint"]?.try &.["commandMetadata"]?.try &.["webCommandMetadata"]? - .try &.["url"]?.try &.as_s? - related_author_url ||= "" - - related_author_thumbnails = renderer.try &.["thumbnail"]?.try &.["thumbnails"]?.try &.as_a? - related_author_thumbnails ||= [] of JSON::Any - - related_author_thumbnail = "" - if related_author_thumbnails.size > 0 - related_author_thumbnail = related_author_thumbnails[-1]["url"]?.try &.as_s? - related_author_thumbnail ||= "" - end - - AboutRelatedChannel.new({ - ucid: related_id, - author: related_title, - author_url: related_author_url, - author_thumbnail: related_author_thumbnail, - }) + tabs = initdata.dig("contents", "twoColumnBrowseResultsRenderer", "tabs").as_a + if tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" } + browse_id = tab.dig?("tabRenderer", "endpoint", "browseEndpoint", "browseId").try(&.as_s?) + params = tab.dig?("tabRenderer", "endpoint", "browseEndpoint", "params").try(&.as_s?) + if browse_id + related_channels = RelatedChannels.new(browse_id: browse_id, params: params) end - related_channels ||= [] of AboutRelatedChannel + end end total_views = 0_i64 @@ -155,20 +129,44 @@ def get_about_info(ucid, locale) sub_count = initdata["header"]["c4TabbedHeaderRenderer"]?.try &.["subscriberCountText"]?.try &.["simpleText"]?.try &.as_s? .try { |text| short_text_to_number(text.split(" ")[0]) } || 0 - AboutChannel.new({ - ucid: ucid, - author: author, - auto_generated: auto_generated, - author_url: author_url, - author_thumbnail: author_thumbnail, - banner: banner, - description_html: description_html, - total_views: total_views, - sub_count: sub_count, - joined: joined, + AboutChannel.new( + ucid: ucid, + author: author, + auto_generated: auto_generated, + author_url: author_url, + author_thumbnail: author_thumbnail, + banner: banner, + description_html: description_html, + total_views: total_views, + sub_count: sub_count, + joined: joined, is_family_friendly: is_family_friendly, - allowed_regions: allowed_regions, - related_channels: related_channels, - tabs: tabs, - }) + allowed_regions: allowed_regions, + related_channels: related_channels, + tabs: tabs, + ) +end + +def fetch_related_channels(related_channels : RelatedChannels) : Array(AboutRelatedChannel) + channels = YoutubeAPI.browse(browse_id: related_channels.browse_id, params: related_channels.params || "") + + tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any + tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" } + return [] of AboutRelatedChannel if tab.nil? + + items = tab.dig?("tabRenderer", "content", "sectionListRenderer", "contents", 0, "itemSectionRenderer", "contents", 0, "gridRenderer", "items").try(&.as_a?) || [] of JSON::Any + + items.map do |item| + related_id = item.dig("gridChannelRenderer", "channelId").as_s + related_title = item.dig("gridChannelRenderer", "title", "simpleText").as_s + related_author_url = item.dig("gridChannelRenderer", "navigationEndpoint", "browseEndpoint", "canonicalBaseUrl").as_s + related_author_thumbnail = item.dig("gridChannelRenderer", "thumbnail", "thumbnails", -1, "url").as_s + + AboutRelatedChannel.new( + ucid: related_id, + author: related_title, + author_url: related_author_url, + author_thumbnail: related_author_thumbnail, + ) + end end diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index 8b6df3fdf..d09641e67 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -96,21 +96,23 @@ module Invidious::Routes::API::V1::Channels json.field "relatedChannels" do json.array do - channel.related_channels.each do |related_channel| - json.object do - json.field "author", related_channel.author - json.field "authorId", related_channel.ucid - json.field "authorUrl", related_channel.author_url + if related_channels = channel.related_channels + fetch_related_channels(related_channels).each do |related_channel| + json.object do + json.field "author", related_channel.author + json.field "authorId", related_channel.ucid + json.field "authorUrl", related_channel.author_url - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} - qualities.each do |quality| - json.object do - json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality + qualities.each do |quality| + json.object do + json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality + end end end end From dc62b6ffdb43bb1a266d05942f3bdfcae7dcf36e Mon Sep 17 00:00:00 2001 From: matthewmcgarvey Date: Mon, 10 Jan 2022 17:07:52 -0600 Subject: [PATCH 2/2] Can fetch related channels using just the channel ucid] --- src/invidious/channels/about.cr | 21 +++---------------- src/invidious/routes/api/v1/channels.cr | 28 ++++++++++++------------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/src/invidious/channels/about.cr b/src/invidious/channels/about.cr index 48f2d4d29..8cae7ae2a 100644 --- a/src/invidious/channels/about.cr +++ b/src/invidious/channels/about.cr @@ -12,13 +12,8 @@ record AboutChannel, joined : Time, is_family_friendly : Bool, allowed_regions : Array(String), - related_channels : RelatedChannels?, tabs : Array(String) -record RelatedChannels, - browse_id : String, - params : String? - record AboutRelatedChannel, ucid : String, author : String, @@ -47,7 +42,6 @@ def get_about_info(ucid, locale) : AboutChannel auto_generated = true end - related_channels = nil if auto_generated author = initdata["header"]["interactiveTabbedHeaderRenderer"]["title"]["simpleText"].as_s author_url = initdata["microformat"]["microformatDataRenderer"]["urlCanonical"].as_s @@ -82,15 +76,6 @@ def get_about_info(ucid, locale) : AboutChannel is_family_friendly = initdata["microformat"]["microformatDataRenderer"]["familySafe"].as_bool allowed_regions = initdata["microformat"]["microformatDataRenderer"]["availableCountries"].as_a.map(&.as_s) - - tabs = initdata.dig("contents", "twoColumnBrowseResultsRenderer", "tabs").as_a - if tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" } - browse_id = tab.dig?("tabRenderer", "endpoint", "browseEndpoint", "browseId").try(&.as_s?) - params = tab.dig?("tabRenderer", "endpoint", "browseEndpoint", "params").try(&.as_s?) - if browse_id - related_channels = RelatedChannels.new(browse_id: browse_id, params: params) - end - end end total_views = 0_i64 @@ -142,13 +127,13 @@ def get_about_info(ucid, locale) : AboutChannel joined: joined, is_family_friendly: is_family_friendly, allowed_regions: allowed_regions, - related_channels: related_channels, tabs: tabs, ) end -def fetch_related_channels(related_channels : RelatedChannels) : Array(AboutRelatedChannel) - channels = YoutubeAPI.browse(browse_id: related_channels.browse_id, params: related_channels.params || "") +def fetch_related_channels(about_channel : AboutChannel) : Array(AboutRelatedChannel) + # params is {"2:string":"channels"} encoded + channels = YoutubeAPI.browse(browse_id: about_channel.ucid, params: "EghjaGFubmVscw%3D%3D") tabs = channels.dig?("contents", "twoColumnBrowseResultsRenderer", "tabs").try(&.as_a?) || [] of JSON::Any tab = tabs.find { |tab| tab.dig?("tabRenderer", "title").try(&.as_s?) == "Channels" } diff --git a/src/invidious/routes/api/v1/channels.cr b/src/invidious/routes/api/v1/channels.cr index d09641e67..322ac42e8 100644 --- a/src/invidious/routes/api/v1/channels.cr +++ b/src/invidious/routes/api/v1/channels.cr @@ -96,23 +96,21 @@ module Invidious::Routes::API::V1::Channels json.field "relatedChannels" do json.array do - if related_channels = channel.related_channels - fetch_related_channels(related_channels).each do |related_channel| - json.object do - json.field "author", related_channel.author - json.field "authorId", related_channel.ucid - json.field "authorUrl", related_channel.author_url + fetch_related_channels(channel).each do |related_channel| + json.object do + json.field "author", related_channel.author + json.field "authorId", related_channel.ucid + json.field "authorUrl", related_channel.author_url - json.field "authorThumbnails" do - json.array do - qualities = {32, 48, 76, 100, 176, 512} + json.field "authorThumbnails" do + json.array do + qualities = {32, 48, 76, 100, 176, 512} - qualities.each do |quality| - json.object do - json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") - json.field "width", quality - json.field "height", quality - end + qualities.each do |quality| + json.object do + json.field "url", related_channel.author_thumbnail.gsub(/=\d+/, "=s#{quality}") + json.field "width", quality + json.field "height", quality end end end