From 1b569bbc99207cae7c20aa285f42477ae361dd30 Mon Sep 17 00:00:00 2001 From: syeopite Date: Sat, 8 May 2021 21:09:30 -0700 Subject: [PATCH] Add support channel home pages + gen. improvements This commit adds support for channel home pages and all of the categories within it. However, the frontend code is a mess and thus needs to be refactor soon. Though that would likely require a rework of items.ecr This commit also comes with some general cleanups and improvements. Before this commit channel brand URls would only be supported on the videos page (now home page). It has been improved to be able to handle all channel URLs. The category_type and auxiliary_data property has also been removed from the Category struct. The former was never used and the latter allows for random data to be added to the Struct presenting documentation issues. Since the auxiliary_data variable was mainly used to store values from the browse_endpoint in order to create URLs, its much simpler to instead just get the URL from the webCommandMetadata. As a result of this change the browse_endpoint_data attribute of Category has also been removed. --- assets/css/channel.css | 21 ++++++- assets/css/default.css | 8 ++- src/invidious.cr | 20 ++----- src/invidious/channels.cr | 42 +++++++++---- src/invidious/helpers/extractors.cr | 38 ++---------- src/invidious/helpers/invidiousitems.cr | 7 +-- src/invidious/routes/channels.cr | 42 ++++++++++++- src/invidious/videos.cr | 2 +- src/invidious/views/channel/channel.ecr | 2 +- src/invidious/views/channel/community.ecr | 2 +- .../views/channel/featured_channels.ecr | 7 +-- src/invidious/views/channel/home.ecr | 60 +++++++++++++++++++ src/invidious/views/channel/playlists.ecr | 2 +- .../views/components/channel-information.ecr | 26 ++++++-- 14 files changed, 197 insertions(+), 82 deletions(-) diff --git a/assets/css/channel.css b/assets/css/channel.css index 85d66a68..c4f03134 100644 --- a/assets/css/channel.css +++ b/assets/css/channel.css @@ -66,7 +66,6 @@ } .category-heading { - font-size: 1.2em; user-select: none; display: inline; } @@ -117,3 +116,23 @@ only show up when the screen is wide enough */ margin-top: 1em; } } + +.trailer-metadata { + margin-left: 15px; + font-size: 12px; + color: rgb(232, 230, 227); +} + +.trailer-metadata .read-more { + line-height: 20px; + text-transform: uppercase; + color: gray; + font-size: 13px; +} + +.trailer-description { + overflow: hidden; + max-height: 150px; + line-height: 20px; + margin-top: 20px; +} \ No newline at end of file diff --git a/assets/css/default.css b/assets/css/default.css index 3434404e..25e4262f 100644 --- a/assets/css/default.css +++ b/assets/css/default.css @@ -602,7 +602,8 @@ hr { } .category { - margin: 3em 0px 4em 0px; + margin-bottom: 2em; + margin-top: 1em; } .category .heading > p { @@ -616,4 +617,9 @@ hr { border-radius: 5px; font-size: 14px; margin-left: 10px; +} + + /* Temp */ +.category-description { + color: #A8A095; } \ No newline at end of file diff --git a/src/invidious.cr b/src/invidious.cr index 0977f3a7..375d2e93 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -315,6 +315,10 @@ Invidious::Routing.get "/channel/:ucid/community", Invidious::Routes::Channels, Invidious::Routing.get "/channel/:ucid/channels", Invidious::Routes::Channels, :channels Invidious::Routing.get "/channel/:ucid/about", Invidious::Routes::Channels, :about +["", "/home", "/videos", "/playlists", "/community", "/channels", "/about"].each do |path| + Invidious::Routing.get "/c/:user#{path}", Invidious::Routes::Channels, :brand_redirect +end + Invidious::Routing.get "/watch", Invidious::Routes::Watch, :handle Invidious::Routing.get "/watch/:id", Invidious::Routes::Watch, :redirect Invidious::Routing.get "/shorts/:id", Invidious::Routes::Watch, :redirect @@ -1624,22 +1628,6 @@ end end end -# YouTube appears to let users set a "brand" URL that -# is different from their username, so we convert that here -get "/c/:user" do |env| - locale = LOCALES[env.get("preferences").as(Preferences).locale]? - - user = env.params.url["user"] - - response = YT_POOL.client &.get("/c/#{user}") - html = XML.parse_html(response.body) - - ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] - next env.redirect "/" if !ucid - - env.redirect "/channel/#{ucid}" -end - # Legacy endpoint for /user/:username get "/profile" do |env| user = env.params.query["user"]? diff --git a/src/invidious/channels.cr b/src/invidious/channels.cr index bdfe5aa6..7e0b1353 100644 --- a/src/invidious/channels.cr +++ b/src/invidious/channels.cr @@ -341,6 +341,30 @@ def fetch_channel(ucid, db, pull_all_videos = true, locale = nil) return channel end +def fetch_channel_home(ucid, channel) + initial_data = request_youtube_api_browse(ucid, channel.tabs["home"][1]) + items = extract_items(initial_data, channel.author, channel.ucid) + + # Channel trailer needs some slight special handling + home_tab = extract_selected_tab(initial_data["contents"]["twoColumnBrowseResultsRenderer"]["tabs"]) + trailer = home_tab["content"]["sectionListRenderer"]["contents"][0]["itemSectionRenderer"]["contents"][0]["channelVideoPlayerRenderer"]? || nil + + home_sections = [] of (Category | Video) + if trailer + trailer = get_video(trailer["videoId"].as_s, PG_DB) + home_sections << trailer + end + + items.each do |category| + if category.is_a? Category + home_sections << category + end + end + + return home_sections + +end + def fetch_channel_playlists(ucid, author, continuation, sort_by) if continuation response_json = request_youtube_api_browse(continuation) @@ -381,8 +405,6 @@ def fetch_channel_playlists(ucid, author, continuation, sort_by) end def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil, continuation = nil, query_title = nil) : {Array(Category), (String | Nil)} - auxiliary_data = {} of String => String - if continuation.is_a?(String) initial_data = request_youtube_api_browse(continuation) items = extract_items(initial_data) @@ -392,14 +414,13 @@ def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil, title: query_title.not_nil!, # If continuation contents is requested then the query_title has to be passed along. contents: items, description_html: "", - browse_endpoint_data: nil, + url: nil, badges: nil, - auxiliary_data: auxiliary_data, })], continuation_token else + url = nil if view && shelf_id - auxiliary_data["view"] = view - auxiliary_data["shelf_id"] = shelf_id + url = "/channel/#{ucid}/channels?view=#{view}&shelf_id=#{shelf_id}" params = produce_featured_channel_browse_param(view.to_i64, shelf_id.to_i64) initial_data = request_youtube_api_browse(ucid, params) @@ -437,21 +458,20 @@ def fetch_channel_featured_channels(ucid, tab_data, view = nil, shelf_id = nil, title: category.title.empty? ? fallback_title : category.title, contents: category.contents, description_html: category.description_html, - browse_endpoint_data: nil, + url: category.url, badges: nil, - auxiliary_data: category.auxiliary_data, }) end - # If we don't have any categories we'll create one. + # If no categories has been parsed then it means that we're currently requesting a single one and not in + # the initial preview anymore. The frontend still needs a Category however, so we'll create one. if category_array.empty? category_array << Category.new({ title: fallback_title, contents: items, description_html: "", - browse_endpoint_data: nil, + url: url, badges: nil, - auxiliary_data: auxiliary_data, }) end diff --git a/src/invidious/helpers/extractors.cr b/src/invidious/helpers/extractors.cr index 430bb41c..9da0bd6b 100644 --- a/src/invidious/helpers/extractors.cr +++ b/src/invidious/helpers/extractors.cr @@ -219,37 +219,7 @@ private class CategoryParser < ItemParser title = "" end - auxiliary_data = {} of String => String - browse_endpoint = item_contents["endpoint"]?.try &.["browseEndpoint"] || nil - browse_endpoint_data = "" - category_type = 0 # 0: Video, 1: Channels, 2: Playlist/feed, 3: trending - - # There's no endpoint data for video and trending category - if !item_contents["endpoint"]? - if !item_contents["videoId"]? - category_type = 3 - end - end - - if !browse_endpoint.nil? - # Playlist/feed categories doesn't need the params value (nor is it even included in yt response) - # instead it uses the browseId parameter. So if there isn't a params value we can assume the - # category is a playlist/feed - if browse_endpoint["params"]? - # However, even though the channel category type returns the browse endpoint param - # we're not going to be using it in order to preserve compatablity with Youtube. - # and for an URL that looks cleaner - url = item_contents["endpoint"]["commandMetadata"]["webCommandMetadata"]["url"] - url = URI.parse(url.as_s) - auxiliary_data["view"] = url.query_params["view"] - auxiliary_data["shelf_id"] = url.query_params["shelf_id"] - - category_type = 1 - else - browse_endpoint_data = browse_endpoint["browseId"].as_s - category_type = 2 - end - end + url = item_contents["endpoint"]?.try &.["commandMetadata"]["webCommandMetadata"]["url"].as_s # Sometimes a category can have badges. badges = [] of Tuple(String, String) # (Badge style, label) @@ -284,9 +254,8 @@ private class CategoryParser < ItemParser title: title, contents: contents, description_html: description_html, - browse_endpoint_data: browse_endpoint_data, + url: url, badges: badges, - auxiliary_data: auxiliary_data, }) end end @@ -325,6 +294,9 @@ private class YoutubeTabsExtractor < ItemsContainerExtractor raw_items << renderer_container_contents next elsif items_container = renderer_container_contents["gridRenderer"]? + elsif items_container = renderer_container_contents["channelVideoPlayerRenderer"]? + # Parsing for channel trailer is already taken elsewhere + next else items_container = renderer_container_contents end diff --git a/src/invidious/helpers/invidiousitems.cr b/src/invidious/helpers/invidiousitems.cr index 83bd5320..9bb7e867 100644 --- a/src/invidious/helpers/invidiousitems.cr +++ b/src/invidious/helpers/invidiousitems.cr @@ -232,14 +232,11 @@ class Category include DB::Serializable property title : String - property contents : Array(SearchItem) - property browse_endpoint_data : String? + property contents : Array(SearchItem) | Array(Video) + property url : String? property description_html : String property badges : Array(Tuple(String, String))? - # Data unique to only specific types of categories. - property auxiliary_data : Hash(String, String) - def to_json(locale, json : JSON::Builder) json.object do json.field "title", self.title diff --git a/src/invidious/routes/channels.cr b/src/invidious/routes/channels.cr index e6e40234..7cd83630 100644 --- a/src/invidious/routes/channels.cr +++ b/src/invidious/routes/channels.cr @@ -1,6 +1,18 @@ class Invidious::Routes::Channels < Invidious::Routes::BaseRoute def home(env) - self.videos(env) + data = self.fetch_basic_information(env) + if !data.is_a?(Tuple) + return data + end + locale, user, subscriptions, continuation, ucid, channel = data + items = fetch_channel_home(ucid, channel) + + has_trailer = false + if items[0].is_a? Video + has_trailer = true + end + + templated "channel/home" end def videos(env) @@ -149,6 +161,34 @@ class Invidious::Routes::Channels < Invidious::Routes::BaseRoute templated "channel/about", buffer_footer: true end + def brand_redirect(env) + locale = LOCALES[env.get("preferences").as(Preferences).locale]? + + user = env.params.url["user"] + + response = YT_POOL.client &.get("/c/#{user}") + html = XML.parse_html(response.body) + + ucid = html.xpath_node(%q(//link[@rel="canonical"])).try &.["href"].split("/")[-1] + if !ucid + env.response.status_code = 404 + return + end + + url = "/channel/#{ucid}" + + location = env.request.path.lchop?("/c/#{user}/") + if location + url += "/#{location}" + end + + if env.params.query.size > 0 + url += "?#{env.params.query}" + end + + env.redirect url + end + private def fetch_basic_information(env) locale = LOCALES[env.get("preferences").as(Preferences).locale]? diff --git a/src/invidious/videos.cr b/src/invidious/videos.cr index 116aafc7..a124dabe 100644 --- a/src/invidious/videos.cr +++ b/src/invidious/videos.cr @@ -275,7 +275,7 @@ struct Video end end - def to_json(locale, json : JSON::Builder) + def to_json(locale : Hash(String, JSON::Any), json : JSON::Builder) json.object do json.field "type", "video" diff --git a/src/invidious/views/channel/channel.ecr b/src/invidious/views/channel/channel.ecr index c0caa80a..1396e397 100644 --- a/src/invidious/views/channel/channel.ecr +++ b/src/invidious/views/channel/channel.ecr @@ -4,7 +4,7 @@ <% end %> -<% content_type = 0 %> +<% content_type = 1 %> <%= rendered "components/channel-information" %>
diff --git a/src/invidious/views/channel/community.ecr b/src/invidious/views/channel/community.ecr index 1b1e8ab2..28183f39 100644 --- a/src/invidious/views/channel/community.ecr +++ b/src/invidious/views/channel/community.ecr @@ -3,7 +3,7 @@ <% end %> -<% content_type = 2 %> +<% content_type = 3 %> <% sort_options = Tuple.new %> <%= rendered "components/channel-information" %> diff --git a/src/invidious/views/channel/featured_channels.ecr b/src/invidious/views/channel/featured_channels.ecr index 2aafa14f..bddac5df 100644 --- a/src/invidious/views/channel/featured_channels.ecr +++ b/src/invidious/views/channel/featured_channels.ecr @@ -13,10 +13,9 @@
-

- <% if category.auxiliary_data.has_key?("view") %> - <% category_url_param = "?view=#{category.auxiliary_data["view"]}&shelf_id=#{category.auxiliary_data["shelf_id"]}" %> - + + <% if category.url %> + <%= category.title %> <%else%> diff --git a/src/invidious/views/channel/home.ecr b/src/invidious/views/channel/home.ecr index e69de29b..ffe15ddf 100644 --- a/src/invidious/views/channel/home.ecr +++ b/src/invidious/views/channel/home.ecr @@ -0,0 +1,60 @@ +<% content_for "header" do %> + <%= channel.author %> - Invidious + +<% end %> + +<% content_type = 0 %> +<% sort_options = Tuple.new %> +<%= rendered "components/channel-information" %> + +
+ <% items.each do | section | %> + <% # Channel trailer %> + <% if section.is_a? Video %> +
+ <% # Placeholder solution. A mini player should be placed here + %> +
+ +
+ + <% if section.length_seconds != 0 %> +

<%= recode_length_seconds(section.length_seconds) %>

+ <% end %> +
+ +
+ +
+ <%= HTML.escape(section.title) %> +

+ <%= translate(locale, "`x` views", number_to_short_text(section.views || 0)) %> + <%= translate(locale, "Shared `x` ago", recode_date(section.published, locale)) %> +

+
+ <%= section.description_html %> +
+ READ MORE +
+
+ <% else %> +
+
+ + <%= section.title %> + + +
+

<%= section.description_html %>

+
+ +
+ <% section.contents.each do |item| %> + <%= rendered "components/item" %> + <% end %> +
+
+
+ <% end %> + <% end %> +
\ No newline at end of file diff --git a/src/invidious/views/channel/playlists.ecr b/src/invidious/views/channel/playlists.ecr index 49dc749e..4883666f 100644 --- a/src/invidious/views/channel/playlists.ecr +++ b/src/invidious/views/channel/playlists.ecr @@ -3,7 +3,7 @@ <% end %> -<% content_type = 1 %> +<% content_type = 2 %> <%= rendered "components/channel-information" %>
diff --git a/src/invidious/views/components/channel-information.ecr b/src/invidious/views/components/channel-information.ecr index b6ec4bb3..9759a8e7 100644 --- a/src/invidious/views/components/channel-information.ecr +++ b/src/invidious/views/components/channel-information.ecr @@ -60,23 +60,37 @@
+ <% if content_type == 0 %> +
  • + + <%= translate(locale, "Home") %> + +
    • + <% else %> +
  • + + <%= translate(locale, "Home") %> + +
    • + <% end %> + <% if !channel.auto_generated %> - <% if content_type == 0 %> + <% if content_type == 1 %>
  • - + <%= translate(locale, "Videos") %>
    • <% else %>
  • - + <%= translate(locale, "Videos") %>
    • <% end %> <% end %> - <% if content_type == 1 || channel.auto_generated %> + <% if content_type == 2 %>
  • <%= translate(locale, "Playlists") %> @@ -91,7 +105,7 @@ <% end %> <% if channel.tabs.has_key?("community") %> - <% if content_type == 2 %> + <% if content_type == 3 %>
  • <%= translate(locale, "Community") %> @@ -152,7 +166,7 @@
    • - <% if content_type == 0 || content_type == 1 %> + <% if content_type == 1 || content_type == 2 %> <% route = content_type == 1 ? "/playlists" : "" %> <% url = "/channel/#{channel.ucid + route}" %>