From 6b929da0e1267a8fc11f2b4154f54b9a407b4580 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Sat, 7 Oct 2023 16:57:47 +0300 Subject: [PATCH 01/28] Added a 'published' video parameter --- src/invidious/jsonify/api_v1/video_json.cr | 1 + src/invidious/videos/parser.cr | 53 ++++++++++++++++++++-- 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index fe4b5223..0deb3204 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -245,6 +245,7 @@ module Invidious::JSONify::APIv1 json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 + json.field "published", rv["published"]?.try { |t| Time.parse(t, "%Y-%m-%d", Time::Location::UTC).to_unix } || Time.utc end end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 06ff96b1..607a418a 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -6,7 +6,7 @@ require "json" # # TODO: "compactRadioRenderer" (Mix) and # TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? +def parse_related_video(related : JSON::Any, published) : Hash(String, JSON::Any)? return nil if !related["videoId"]? # The compact renderer has video length in seconds, where the end @@ -47,6 +47,48 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? "view_count" => JSON::Any.new(view_count || "0"), "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), + "published" => published, + } +end + +def fetch_published(video_id : String, proxy_region : String? = nil) + client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) + player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) + playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s + if playability_status != "OK" + subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") + reason = subreason.try &.[]?("simpleText").try &.as_s + reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") + reason ||= player_response.dig("playabilityStatus", "reason").as_s + + # Stop here if video is not a scheduled livestream or + # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help + if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) || + playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails") + return { + "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), + "reason" => JSON::Any.new(reason), + } + end + elsif video_id != player_response.dig("videoDetails", "videoId") + # YouTube may return a different video player response than expected. + # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 + # Line to be reverted if one day we solve the video not available issue. + return { + "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), + "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."), + } + else + reason = nil + end + microformat1 = player_response.dig?("microformat", "playerMicroformatRenderer") + if !microformat1.nil? + published1 = microformat1["publishDate"] + else + published1 = JSON::Any.new("") + end + return { + "published" => published1, } end @@ -92,7 +134,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) player_response = player_response.merge(next_response) end - params = parse_video_info(video_id, player_response) + params = parse_video_info(video_id, player_response, proxy_region) params["reason"] = JSON::Any.new(reason) if reason new_player_response = nil @@ -157,7 +199,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf end end -def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) +def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any), proxy_region : String? = nil) : Hash(String, JSON::Any) # Top level elements main_results = player_response.dig?("contents", "twoColumnWatchNextResults") @@ -235,7 +277,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - related_video = parse_related_video(item) + published1 = fetch_published(item["videoId"].to_s, proxy_region)["published"] + related_video = parse_related_video(item, published1) related << JSON::Any.new(related_video) if related_video end end @@ -250,7 +293,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any player_overlays.try &.as_a.each do |element| if item = element["endScreenVideoRenderer"]? - related_video = parse_related_video(item) + related_video = parse_related_video(item, JSON::Any.new("")) related << JSON::Any.new(related_video) if related_video end end From b7a252b096f7bc29f105ea25454fe053cf4b4d4b Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Mon, 9 Oct 2023 12:00:37 +0300 Subject: [PATCH 02/28] Removed need for more API calls by parsing the publishedTimeText string --- src/invidious/jsonify/api_v1/video_json.cr | 2 +- src/invidious/videos/parser.cr | 99 +++++++++++++--------- 2 files changed, 58 insertions(+), 43 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 0deb3204..8a74343c 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -245,7 +245,7 @@ module Invidious::JSONify::APIv1 json.field "lengthSeconds", rv["length_seconds"]?.try &.to_i json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 - json.field "published", rv["published"]?.try { |t| Time.parse(t, "%Y-%m-%d", Time::Location::UTC).to_unix } || Time.utc + json.field "published", rv["published"]? end end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 607a418a..a1b9b988 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -51,47 +51,6 @@ def parse_related_video(related : JSON::Any, published) : Hash(String, JSON::Any } end -def fetch_published(video_id : String, proxy_region : String? = nil) - client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) - player_response = YoutubeAPI.player(video_id: video_id, params: "", client_config: client_config) - playability_status = player_response.dig?("playabilityStatus", "status").try &.as_s - if playability_status != "OK" - subreason = player_response.dig?("playabilityStatus", "errorScreen", "playerErrorMessageRenderer", "subreason") - reason = subreason.try &.[]?("simpleText").try &.as_s - reason ||= subreason.try &.[]("runs").as_a.map(&.[]("text")).join("") - reason ||= player_response.dig("playabilityStatus", "reason").as_s - - # Stop here if video is not a scheduled livestream or - # for LOGIN_REQUIRED when videoDetails element is not found because retrying won't help - if !{"LIVE_STREAM_OFFLINE", "LOGIN_REQUIRED"}.any?(playability_status) || - playability_status == "LOGIN_REQUIRED" && !player_response.dig?("videoDetails") - return { - "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), - "reason" => JSON::Any.new(reason), - } - end - elsif video_id != player_response.dig("videoDetails", "videoId") - # YouTube may return a different video player response than expected. - # See: https://github.com/TeamNewPipe/NewPipe/issues/8713 - # Line to be reverted if one day we solve the video not available issue. - return { - "version" => JSON::Any.new(Video::SCHEMA_VERSION.to_i64), - "reason" => JSON::Any.new("Can't load the video on this Invidious instance. YouTube is currently trying to block Invidious instances. Click here for more info about the issue."), - } - else - reason = nil - end - microformat1 = player_response.dig?("microformat", "playerMicroformatRenderer") - if !microformat1.nil? - published1 = microformat1["publishDate"] - else - published1 = JSON::Any.new("") - end - return { - "published" => published1, - } -end - def extract_video_info(video_id : String, proxy_region : String? = nil) # Init client config for the API client_config = YoutubeAPI::ClientConfig.new(proxy_region: proxy_region) @@ -277,7 +236,63 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - published1 = fetch_published(item["videoId"].to_s, proxy_region)["published"] + time_string = item["publishedTimeText"]["simpleText"]? + if !time_string.nil? && time_string.to_s.ends_with?("hour ago") + time = Time.utc.to_unix - 3600 + end + if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && !time_string.to_s.starts_with?("Streamed") + hours = time_string.to_s.rchop(" hours ago").to_i + time = Time.utc.to_unix - 3600*hours + end + if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && time_string.to_s.starts_with?("Streamed") + hours = time_string.to_s.lchop("Streamed ").rchop(" hours ago").to_i + time = Time.utc.to_unix - 3600*hours + end + if !time_string.nil? && time_string.to_s.ends_with?("day ago") + time = Time.utc.to_unix - 86400 + end + if !time_string.nil? && time_string.to_s.ends_with?("days ago") && !time_string.to_s.starts_with?("Streamed") + days = time_string.to_s.rchop(" days ago").to_i + time = Time.utc.to_unix - 86400*days + end + if !time_string.nil? && time_string.to_s.ends_with?("days ago") && time_string.to_s.starts_with?("Streamed") + days = time_string.to_s.lchop("Streamed ").rchop(" days ago").to_i + time = Time.utc.to_unix - 86400*days + end + if !time_string.nil? && time_string.to_s.ends_with?("week ago") + time = Time.utc.to_unix - 604800 + end + if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && !time_string.to_s.starts_with?("Streamed") + weeks = time_string.to_s.rchop(" weeks ago").to_i + time = Time.utc.to_unix - 604800*weeks + end + if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && time_string.to_s.starts_with?("Streamed") + weeks = time_string.to_s.lchop("Streamed ").rchop(" weeks ago").to_i + time = Time.utc.to_unix - 604800*weeks + end + if !time_string.nil? && time_string.to_s.ends_with?("month ago") + time = Time.utc.to_unix - 2629743 + end + if !time_string.nil? && time_string.to_s.ends_with?("months ago") && !time_string.to_s.starts_with?("Streamed") + months = time_string.to_s.rchop(" months ago").to_i + time = Time.utc.to_unix - 2629743*months + end + if !time_string.nil? && time_string.to_s.ends_with?("months ago") && time_string.to_s.starts_with?("Streamed") + months = time_string.to_s.lchop("Streamed ").rchop(" months ago").to_i + time = Time.utc.to_unix - 2629743*months + end + if !time_string.nil? && time_string.to_s.ends_with?("year ago") + time = Time.utc.to_unix - 31556926 + end + if !time_string.nil? && time_string.to_s.ends_with?("years ago") && !time_string.to_s.starts_with?("Streamed") + years = time_string.to_s.rchop(" years ago").to_i + time = Time.utc.to_unix - 31556926*years + end + if !time_string.nil? && time_string.to_s.ends_with?("years ago") && time_string.to_s.starts_with?("Streamed") + years = time_string.to_s.lchop("Streamed ").rchop(" years ago").to_i + time = Time.utc.to_unix - 31556926*years + end + published1 = JSON::Any.new(time.to_s) related_video = parse_related_video(item, published1) related << JSON::Any.new(related_video) if related_video end From f9460e31bc77d17c7fc6719770402008b422d7b4 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Mon, 9 Oct 2023 12:09:03 +0300 Subject: [PATCH 03/28] Fixed an issue --- src/invidious/videos/parser.cr | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index a1b9b988..e7458672 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -236,7 +236,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - time_string = item["publishedTimeText"]["simpleText"]? + time_text = item["publishedTimeText"]? + time_string = nil + if !time_text.nil? + time_string = time_text["simpleText"]? + end if !time_string.nil? && time_string.to_s.ends_with?("hour ago") time = Time.utc.to_unix - 3600 end From 48af0af9d5e0ca873fb216e8bf88eadb18428a7e Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Mon, 9 Oct 2023 12:18:50 +0300 Subject: [PATCH 04/28] Added minutes as well --- src/invidious/videos/parser.cr | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index e7458672..1b627c62 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -241,6 +241,17 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any if !time_text.nil? time_string = time_text["simpleText"]? end + if !time_string.nil? && time_string.to_s.ends_with?("minute ago") + time = Time.utc.to_unix - 60 + end + if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && !time_string.to_s.starts_with?("Streamed") + minutes = time_string.to_s.rchop(" minutes ago").to_i + time = Time.utc.to_unix - 60*minutes + end + if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && time_string.to_s.starts_with?("Streamed") + minutes = time_string.to_s.lchop("Streamed ").rchop(" minutes ago").to_i + time = Time.utc.to_unix - 60*minutes + end if !time_string.nil? && time_string.to_s.ends_with?("hour ago") time = Time.utc.to_unix - 3600 end From b0b4f09b3a3638a97fd6d7845dad1638e56490ae Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Mon, 9 Oct 2023 12:26:38 +0300 Subject: [PATCH 05/28] Seperated the code in a function --- src/invidious/videos/parser.cr | 151 +++++++++++++++++---------------- 1 file changed, 80 insertions(+), 71 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 1b627c62..5444088b 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -158,6 +158,85 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf end end +def parse_published_string(item) + time_text = item["publishedTimeText"]? + time_string = nil + if !time_text.nil? + time_string = time_text["simpleText"]? + end + if !time_string.nil? && time_string.to_s.ends_with?("minute ago") + time = Time.utc.to_unix - 60 + end + if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && !time_string.to_s.starts_with?("Streamed") + minutes = time_string.to_s.rchop(" minutes ago").to_i + time = Time.utc.to_unix - 60*minutes + end + if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && time_string.to_s.starts_with?("Streamed") + minutes = time_string.to_s.lchop("Streamed ").rchop(" minutes ago").to_i + time = Time.utc.to_unix - 60*minutes + end + if !time_string.nil? && time_string.to_s.ends_with?("hour ago") + time = Time.utc.to_unix - 3600 + end + if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && !time_string.to_s.starts_with?("Streamed") + hours = time_string.to_s.rchop(" hours ago").to_i + time = Time.utc.to_unix - 3600*hours + end + if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && time_string.to_s.starts_with?("Streamed") + hours = time_string.to_s.lchop("Streamed ").rchop(" hours ago").to_i + time = Time.utc.to_unix - 3600*hours + end + if !time_string.nil? && time_string.to_s.ends_with?("day ago") + time = Time.utc.to_unix - 86400 + end + if !time_string.nil? && time_string.to_s.ends_with?("days ago") && !time_string.to_s.starts_with?("Streamed") + days = time_string.to_s.rchop(" days ago").to_i + time = Time.utc.to_unix - 86400*days + end + if !time_string.nil? && time_string.to_s.ends_with?("days ago") && time_string.to_s.starts_with?("Streamed") + days = time_string.to_s.lchop("Streamed ").rchop(" days ago").to_i + time = Time.utc.to_unix - 86400*days + end + if !time_string.nil? && time_string.to_s.ends_with?("week ago") + time = Time.utc.to_unix - 604800 + end + if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && !time_string.to_s.starts_with?("Streamed") + weeks = time_string.to_s.rchop(" weeks ago").to_i + time = Time.utc.to_unix - 604800*weeks + end + if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && time_string.to_s.starts_with?("Streamed") + weeks = time_string.to_s.lchop("Streamed ").rchop(" weeks ago").to_i + time = Time.utc.to_unix - 604800*weeks + end + if !time_string.nil? && time_string.to_s.ends_with?("month ago") + time = Time.utc.to_unix - 2629743 + end + if !time_string.nil? && time_string.to_s.ends_with?("months ago") && !time_string.to_s.starts_with?("Streamed") + months = time_string.to_s.rchop(" months ago").to_i + time = Time.utc.to_unix - 2629743*months + end + if !time_string.nil? && time_string.to_s.ends_with?("months ago") && time_string.to_s.starts_with?("Streamed") + months = time_string.to_s.lchop("Streamed ").rchop(" months ago").to_i + time = Time.utc.to_unix - 2629743*months + end + if !time_string.nil? && time_string.to_s.ends_with?("year ago") + time = Time.utc.to_unix - 31556926 + end + if !time_string.nil? && time_string.to_s.ends_with?("years ago") && !time_string.to_s.starts_with?("Streamed") + years = time_string.to_s.rchop(" years ago").to_i + time = Time.utc.to_unix - 31556926*years + end + if !time_string.nil? && time_string.to_s.ends_with?("years ago") && time_string.to_s.starts_with?("Streamed") + years = time_string.to_s.lchop("Streamed ").rchop(" years ago").to_i + time = Time.utc.to_unix - 31556926*years + end + if time_string.nil? + time = nil + end + + return time +end + def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any), proxy_region : String? = nil) : Hash(String, JSON::Any) # Top level elements @@ -236,77 +315,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - time_text = item["publishedTimeText"]? - time_string = nil - if !time_text.nil? - time_string = time_text["simpleText"]? - end - if !time_string.nil? && time_string.to_s.ends_with?("minute ago") - time = Time.utc.to_unix - 60 - end - if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && !time_string.to_s.starts_with?("Streamed") - minutes = time_string.to_s.rchop(" minutes ago").to_i - time = Time.utc.to_unix - 60*minutes - end - if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && time_string.to_s.starts_with?("Streamed") - minutes = time_string.to_s.lchop("Streamed ").rchop(" minutes ago").to_i - time = Time.utc.to_unix - 60*minutes - end - if !time_string.nil? && time_string.to_s.ends_with?("hour ago") - time = Time.utc.to_unix - 3600 - end - if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && !time_string.to_s.starts_with?("Streamed") - hours = time_string.to_s.rchop(" hours ago").to_i - time = Time.utc.to_unix - 3600*hours - end - if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && time_string.to_s.starts_with?("Streamed") - hours = time_string.to_s.lchop("Streamed ").rchop(" hours ago").to_i - time = Time.utc.to_unix - 3600*hours - end - if !time_string.nil? && time_string.to_s.ends_with?("day ago") - time = Time.utc.to_unix - 86400 - end - if !time_string.nil? && time_string.to_s.ends_with?("days ago") && !time_string.to_s.starts_with?("Streamed") - days = time_string.to_s.rchop(" days ago").to_i - time = Time.utc.to_unix - 86400*days - end - if !time_string.nil? && time_string.to_s.ends_with?("days ago") && time_string.to_s.starts_with?("Streamed") - days = time_string.to_s.lchop("Streamed ").rchop(" days ago").to_i - time = Time.utc.to_unix - 86400*days - end - if !time_string.nil? && time_string.to_s.ends_with?("week ago") - time = Time.utc.to_unix - 604800 - end - if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && !time_string.to_s.starts_with?("Streamed") - weeks = time_string.to_s.rchop(" weeks ago").to_i - time = Time.utc.to_unix - 604800*weeks - end - if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && time_string.to_s.starts_with?("Streamed") - weeks = time_string.to_s.lchop("Streamed ").rchop(" weeks ago").to_i - time = Time.utc.to_unix - 604800*weeks - end - if !time_string.nil? && time_string.to_s.ends_with?("month ago") - time = Time.utc.to_unix - 2629743 - end - if !time_string.nil? && time_string.to_s.ends_with?("months ago") && !time_string.to_s.starts_with?("Streamed") - months = time_string.to_s.rchop(" months ago").to_i - time = Time.utc.to_unix - 2629743*months - end - if !time_string.nil? && time_string.to_s.ends_with?("months ago") && time_string.to_s.starts_with?("Streamed") - months = time_string.to_s.lchop("Streamed ").rchop(" months ago").to_i - time = Time.utc.to_unix - 2629743*months - end - if !time_string.nil? && time_string.to_s.ends_with?("year ago") - time = Time.utc.to_unix - 31556926 - end - if !time_string.nil? && time_string.to_s.ends_with?("years ago") && !time_string.to_s.starts_with?("Streamed") - years = time_string.to_s.rchop(" years ago").to_i - time = Time.utc.to_unix - 31556926*years - end - if !time_string.nil? && time_string.to_s.ends_with?("years ago") && time_string.to_s.starts_with?("Streamed") - years = time_string.to_s.lchop("Streamed ").rchop(" years ago").to_i - time = Time.utc.to_unix - 31556926*years - end + time = parse_published_string(item) published1 = JSON::Any.new(time.to_s) related_video = parse_related_video(item, published1) related << JSON::Any.new(related_video) if related_video From 20ca1ebcc0e0c6a7041dfa12dd241abd2b3f464c Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Wed, 11 Oct 2023 09:08:23 +0300 Subject: [PATCH 06/28] Used the decode_date function instead --- src/invidious/videos/parser.cr | 83 +--------------------------------- 1 file changed, 2 insertions(+), 81 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 5444088b..ed313fe5 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -158,85 +158,6 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf end end -def parse_published_string(item) - time_text = item["publishedTimeText"]? - time_string = nil - if !time_text.nil? - time_string = time_text["simpleText"]? - end - if !time_string.nil? && time_string.to_s.ends_with?("minute ago") - time = Time.utc.to_unix - 60 - end - if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && !time_string.to_s.starts_with?("Streamed") - minutes = time_string.to_s.rchop(" minutes ago").to_i - time = Time.utc.to_unix - 60*minutes - end - if !time_string.nil? && time_string.to_s.ends_with?("minutes ago") && time_string.to_s.starts_with?("Streamed") - minutes = time_string.to_s.lchop("Streamed ").rchop(" minutes ago").to_i - time = Time.utc.to_unix - 60*minutes - end - if !time_string.nil? && time_string.to_s.ends_with?("hour ago") - time = Time.utc.to_unix - 3600 - end - if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && !time_string.to_s.starts_with?("Streamed") - hours = time_string.to_s.rchop(" hours ago").to_i - time = Time.utc.to_unix - 3600*hours - end - if !time_string.nil? && time_string.to_s.ends_with?("hours ago") && time_string.to_s.starts_with?("Streamed") - hours = time_string.to_s.lchop("Streamed ").rchop(" hours ago").to_i - time = Time.utc.to_unix - 3600*hours - end - if !time_string.nil? && time_string.to_s.ends_with?("day ago") - time = Time.utc.to_unix - 86400 - end - if !time_string.nil? && time_string.to_s.ends_with?("days ago") && !time_string.to_s.starts_with?("Streamed") - days = time_string.to_s.rchop(" days ago").to_i - time = Time.utc.to_unix - 86400*days - end - if !time_string.nil? && time_string.to_s.ends_with?("days ago") && time_string.to_s.starts_with?("Streamed") - days = time_string.to_s.lchop("Streamed ").rchop(" days ago").to_i - time = Time.utc.to_unix - 86400*days - end - if !time_string.nil? && time_string.to_s.ends_with?("week ago") - time = Time.utc.to_unix - 604800 - end - if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && !time_string.to_s.starts_with?("Streamed") - weeks = time_string.to_s.rchop(" weeks ago").to_i - time = Time.utc.to_unix - 604800*weeks - end - if !time_string.nil? && time_string.to_s.ends_with?("weeks ago") && time_string.to_s.starts_with?("Streamed") - weeks = time_string.to_s.lchop("Streamed ").rchop(" weeks ago").to_i - time = Time.utc.to_unix - 604800*weeks - end - if !time_string.nil? && time_string.to_s.ends_with?("month ago") - time = Time.utc.to_unix - 2629743 - end - if !time_string.nil? && time_string.to_s.ends_with?("months ago") && !time_string.to_s.starts_with?("Streamed") - months = time_string.to_s.rchop(" months ago").to_i - time = Time.utc.to_unix - 2629743*months - end - if !time_string.nil? && time_string.to_s.ends_with?("months ago") && time_string.to_s.starts_with?("Streamed") - months = time_string.to_s.lchop("Streamed ").rchop(" months ago").to_i - time = Time.utc.to_unix - 2629743*months - end - if !time_string.nil? && time_string.to_s.ends_with?("year ago") - time = Time.utc.to_unix - 31556926 - end - if !time_string.nil? && time_string.to_s.ends_with?("years ago") && !time_string.to_s.starts_with?("Streamed") - years = time_string.to_s.rchop(" years ago").to_i - time = Time.utc.to_unix - 31556926*years - end - if !time_string.nil? && time_string.to_s.ends_with?("years ago") && time_string.to_s.starts_with?("Streamed") - years = time_string.to_s.lchop("Streamed ").rchop(" years ago").to_i - time = Time.utc.to_unix - 31556926*years - end - if time_string.nil? - time = nil - end - - return time -end - def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any), proxy_region : String? = nil) : Hash(String, JSON::Any) # Top level elements @@ -315,8 +236,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - time = parse_published_string(item) - published1 = JSON::Any.new(time.to_s) + time = decode_date(item["publishedTimeText"].to_s) + published1 = JSON::Any.new(time.to_unix.to_s) related_video = parse_related_video(item, published1) related << JSON::Any.new(related_video) if related_video end From fa59f41f7bc4df72ea49d6e2e9902a93e0de5228 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Wed, 11 Oct 2023 09:12:27 +0300 Subject: [PATCH 07/28] Fixed an issue --- src/invidious/videos/parser.cr | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index ed313fe5..373715fe 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -236,8 +236,13 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - time = decode_date(item["publishedTimeText"].to_s) - published1 = JSON::Any.new(time.to_unix.to_s) + time_text = item["publishedTimeText"]? + if !time_text.nil? + time = decode_date(item["publishedTimeText"].to_s) + published1 = JSON::Any.new(time.to_unix.to_s) + else + published1 = JSON::Any.new("") + end related_video = parse_related_video(item, published1) related << JSON::Any.new(related_video) if related_video end From 6236cea33e1393a4708bfd3a0083e05e553bc8ed Mon Sep 17 00:00:00 2001 From: RadoslavL <63926126+RadoslavL@users.noreply.github.com> Date: Wed, 8 Nov 2023 10:13:16 +0200 Subject: [PATCH 08/28] Changed some variable names Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- src/invidious/videos/parser.cr | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 373715fe..52623051 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -236,14 +236,13 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - time_text = item["publishedTimeText"]? - if !time_text.nil? - time = decode_date(item["publishedTimeText"].to_s) - published1 = JSON::Any.new(time.to_unix.to_s) + if item["publishedTimeText"]? + rv_decoded_time = decode_date(item["publishedTimeText"].to_s) + rv_published_timestamp = JSON::Any.new(rv_decoded_time.to_unix.to_s) else - published1 = JSON::Any.new("") + rv_published_timestamp = JSON::Any.new("") end - related_video = parse_related_video(item, published1) + related_video = parse_related_video(item, rv_published_timestamp) related << JSON::Any.new(related_video) if related_video end end From 76369eb599ff3ae8a1fd89e6c2d1ae745b1276e3 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Wed, 8 Nov 2023 10:18:29 +0200 Subject: [PATCH 09/28] Removed unused attribute --- src/invidious/videos/parser.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 52623051..275395df 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -93,7 +93,7 @@ def extract_video_info(video_id : String, proxy_region : String? = nil) player_response = player_response.merge(next_response) end - params = parse_video_info(video_id, player_response, proxy_region) + params = parse_video_info(video_id, player_response) params["reason"] = JSON::Any.new(reason) if reason new_player_response = nil @@ -158,7 +158,7 @@ def try_fetch_streaming_data(id : String, client_config : YoutubeAPI::ClientConf end end -def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any), proxy_region : String? = nil) : Hash(String, JSON::Any) +def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any)) : Hash(String, JSON::Any) # Top level elements main_results = player_response.dig?("contents", "twoColumnWatchNextResults") From a0d24190b8e7b4e92f4a4912953cace1fc52f43d Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Wed, 8 Nov 2023 19:09:16 +0200 Subject: [PATCH 10/28] Made published be an optional parameter --- src/invidious/videos/parser.cr | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 275395df..c0e8a6ea 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -6,7 +6,7 @@ require "json" # # TODO: "compactRadioRenderer" (Mix) and # TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any, published) : Hash(String, JSON::Any)? +def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(String, JSON::Any)? return nil if !related["videoId"]? # The compact renderer has video length in seconds, where the end @@ -47,7 +47,7 @@ def parse_related_video(related : JSON::Any, published) : Hash(String, JSON::Any "view_count" => JSON::Any.new(view_count || "0"), "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), - "published" => published, + "published" => JSON::Any.new(published || ""), } end @@ -238,11 +238,11 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any if item = element["compactVideoRenderer"]? if item["publishedTimeText"]? rv_decoded_time = decode_date(item["publishedTimeText"].to_s) - rv_published_timestamp = JSON::Any.new(rv_decoded_time.to_unix.to_s) + rv_published_timestamp = rv_decoded_time.to_unix.to_s else - rv_published_timestamp = JSON::Any.new("") + rv_published_timestamp = nil end - related_video = parse_related_video(item, rv_published_timestamp) + related_video = parse_related_video(item, published: rv_published_timestamp) related << JSON::Any.new(related_video) if related_video end end @@ -257,7 +257,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any player_overlays.try &.as_a.each do |element| if item = element["endScreenVideoRenderer"]? - related_video = parse_related_video(item, JSON::Any.new("")) + related_video = parse_related_video(item) related << JSON::Any.new(related_video) if related_video end end From be216fff94016ed5465b43ad6ef0c589290e7e21 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Sun, 12 Nov 2023 08:37:13 +0200 Subject: [PATCH 11/28] Added the text version of the published parameter --- src/invidious/jsonify/api_v1/video_json.cr | 1 + src/invidious/videos/parser.cr | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 8a74343c..79f3d46f 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -246,6 +246,7 @@ module Invidious::JSONify::APIv1 json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? + json.field "publishedTimeText", rv["publishedText"]? end end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index c0e8a6ea..e8988587 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -6,7 +6,7 @@ require "json" # # TODO: "compactRadioRenderer" (Mix) and # TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(String, JSON::Any)? +def parse_related_video(related : JSON::Any, published : String? = nil, publishedText : String? = nil) : Hash(String, JSON::Any)? return nil if !related["videoId"]? # The compact renderer has video length in seconds, where the end @@ -48,6 +48,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), + "publishedText" => JSON::Any.new(publishedText || ""), } end @@ -237,12 +238,13 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? if item["publishedTimeText"]? + rv_published_time_text = item["publishedTimeText"].to_s rv_decoded_time = decode_date(item["publishedTimeText"].to_s) rv_published_timestamp = rv_decoded_time.to_unix.to_s else rv_published_timestamp = nil end - related_video = parse_related_video(item, published: rv_published_timestamp) + related_video = parse_related_video(item, published: rv_published_timestamp, publishedText: rv_published_time_text) related << JSON::Any.new(related_video) if related_video end end From 7388e4ca72dd55e329b5105959fe215db0baf7c0 Mon Sep 17 00:00:00 2001 From: RadoslavL <63926126+RadoslavL@users.noreply.github.com> Date: Sun, 12 Nov 2023 20:51:33 +0200 Subject: [PATCH 12/28] Add translation to the `publishedText` parameter Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- src/invidious/videos/parser.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index e8988587..82164e3c 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -48,7 +48,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil, publishe "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), - "publishedText" => JSON::Any.new(publishedText || ""), + "publishedText" => JSON::Any.new(translate(locale, "`x` ago", recode_date(video.published, locale)) || ""), } end From 50da6cf3e760950b35c16600104980f0b909e8dc Mon Sep 17 00:00:00 2001 From: RadoslavL <63926126+RadoslavL@users.noreply.github.com> Date: Sun, 12 Nov 2023 20:52:11 +0200 Subject: [PATCH 13/28] Organize the code better Co-authored-by: syeopite <70992037+syeopite@users.noreply.github.com> --- src/invidious/videos/parser.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 82164e3c..929d492f 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -237,9 +237,9 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - if item["publishedTimeText"]? - rv_published_time_text = item["publishedTimeText"].to_s - rv_decoded_time = decode_date(item["publishedTimeText"].to_s) + if rv_published_time_text = item["publishedTimeText"]? + rv_published_time_text = rv_published_time_text.as_s + rv_decoded_time = decode_date(rv_published_time_text) rv_published_timestamp = rv_decoded_time.to_unix.to_s else rv_published_timestamp = nil From 2a6a32e667f5e9ed868c275531ff63c113429321 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Tue, 14 Nov 2023 09:43:52 +0200 Subject: [PATCH 14/28] Fixed an issue --- src/invidious/jsonify/api_v1/video_json.cr | 2 +- src/invidious/videos/parser.cr | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 79f3d46f..f5450f3f 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -246,7 +246,7 @@ module Invidious::JSONify::APIv1 json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? - json.field "publishedTimeText", rv["publishedText"]? + json.field "publishedTimeText", translate(locale, "`x` ago", rv["publishedText"].to_s) end end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 929d492f..b96f4279 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -6,7 +6,7 @@ require "json" # # TODO: "compactRadioRenderer" (Mix) and # TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any, published : String? = nil, publishedText : String? = nil) : Hash(String, JSON::Any)? +def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(String, JSON::Any)? return nil if !related["videoId"]? # The compact renderer has video length in seconds, where the end @@ -36,6 +36,8 @@ def parse_related_video(related : JSON::Any, published : String? = nil, publishe LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") + publishedText = related["publishedTimeText"]["simpleText"].to_s + # TODO: when refactoring video types, make a struct for related videos # or reuse an existing type, if that fits. return { @@ -48,7 +50,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil, publishe "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), - "publishedText" => JSON::Any.new(translate(locale, "`x` ago", recode_date(video.published, locale)) || ""), + "publishedText" => JSON::Any.new(publishedText || ""), } end @@ -237,14 +239,13 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - if rv_published_time_text = item["publishedTimeText"]? - rv_published_time_text = rv_published_time_text.as_s - rv_decoded_time = decode_date(rv_published_time_text) + if rv_published_time_text = item["publishedTimeText"]["simpleText"]? + rv_decoded_time = decode_date(rv_published_time_text.to_s) rv_published_timestamp = rv_decoded_time.to_unix.to_s else rv_published_timestamp = nil end - related_video = parse_related_video(item, published: rv_published_timestamp, publishedText: rv_published_time_text) + related_video = parse_related_video(item, published: rv_published_timestamp) related << JSON::Any.new(related_video) if related_video end end From 0d22af65649a0798531251949ff21954c0780643 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Tue, 14 Nov 2023 09:47:16 +0200 Subject: [PATCH 15/28] Moved methods around --- src/invidious/videos/parser.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index b96f4279..f29c81e3 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -239,8 +239,8 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - if rv_published_time_text = item["publishedTimeText"]["simpleText"]? - rv_decoded_time = decode_date(rv_published_time_text.to_s) + if rv_published_time_text = item["publishedTimeText"]? + rv_decoded_time = decode_date(rv_published_time_text["simpleText"].to_s) rv_published_timestamp = rv_decoded_time.to_unix.to_s else rv_published_timestamp = nil From 3bced4e12b6ecf26e0b3dfdb72aec8cf89197803 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Tue, 14 Nov 2023 09:51:12 +0200 Subject: [PATCH 16/28] Fixed another issue --- src/invidious/videos/parser.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index f29c81e3..83c97602 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -36,7 +36,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") - publishedText = related["publishedTimeText"]["simpleText"].to_s + publishedText = related["publishedTimeText"]? # TODO: when refactoring video types, make a struct for related videos # or reuse an existing type, if that fits. @@ -50,7 +50,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), - "publishedText" => JSON::Any.new(publishedText || ""), + "publishedText" => JSON::Any.new(publishedText["simpleText"]?.to_s || ""), } end From 4c486634e26616a253e907ced0fe169e9a8de284 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Tue, 14 Nov 2023 09:56:06 +0200 Subject: [PATCH 17/28] Another attempt at fixing the issue --- src/invidious/videos/parser.cr | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 83c97602..dc9962bc 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -37,6 +37,11 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") publishedText = related["publishedTimeText"]? + if !publishedText.nil? + publishedSimpleText = publishedText["simpleText"] + else + publishedSimpleText = nil + end # TODO: when refactoring video types, make a struct for related videos # or reuse an existing type, if that fits. @@ -50,7 +55,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), - "publishedText" => JSON::Any.new(publishedText["simpleText"]?.to_s || ""), + "publishedText" => JSON::Any.new(publishedSimpleText || ""), } end From d098e5ae9ba3058846c9fd0040c18f5a3a146bd8 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Tue, 14 Nov 2023 09:58:37 +0200 Subject: [PATCH 18/28] I hope it works at this point --- src/invidious/videos/parser.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index dc9962bc..00c9c8a3 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -38,7 +38,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S publishedText = related["publishedTimeText"]? if !publishedText.nil? - publishedSimpleText = publishedText["simpleText"] + publishedSimpleText = publishedText["simpleText"].to_s else publishedSimpleText = nil end From 03f9962a479884d8c5cd86c56caef728a7744c0e Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Tue, 14 Nov 2023 10:00:18 +0200 Subject: [PATCH 19/28] This should work --- src/invidious/videos/parser.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 00c9c8a3..279daf37 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -38,7 +38,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S publishedText = related["publishedTimeText"]? if !publishedText.nil? - publishedSimpleText = publishedText["simpleText"].to_s + publishedSimpleText = publishedText["simpleText"].to_s else publishedSimpleText = nil end @@ -55,7 +55,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), - "publishedText" => JSON::Any.new(publishedSimpleText || ""), + "publishedText" => JSON::Any.new(publishedSimpleText || ""), } end From 68611482906f3b70e9c105fec5c57436c877a3b0 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Fri, 24 Nov 2023 11:24:56 +0200 Subject: [PATCH 20/28] Moved code around and fixed a problem --- src/invidious/jsonify/api_v1/video_json.cr | 2 +- src/invidious/videos/parser.cr | 21 ++++++++------------- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index f5450f3f..622641ee 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -246,7 +246,7 @@ module Invidious::JSONify::APIv1 json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? - json.field "publishedTimeText", translate(locale, "`x` ago", rv["publishedText"].to_s) + json.field "publishedTimeText", translate(locale, "`x` ago", rv["publishedText"].to_s.gsub(" ago", "")) end end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 279daf37..9275b1bb 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -6,7 +6,7 @@ require "json" # # TODO: "compactRadioRenderer" (Mix) and # TODO: Use a proper struct/class instead of a hacky JSON object -def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(String, JSON::Any)? +def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? return nil if !related["videoId"]? # The compact renderer has video length in seconds, where the end @@ -36,11 +36,12 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S LOGGER.trace("parse_related_video: Found \"watchNextEndScreenRenderer\" container") - publishedText = related["publishedTimeText"]? - if !publishedText.nil? - publishedSimpleText = publishedText["simpleText"].to_s + if published_time_text = related["publishedTimeText"]? + decoded_time = decode_date(published_time_text["simpleText"].to_s) + published = decoded_time.to_unix.to_s + published_time_text = published_time_text["simpleText"].to_s else - publishedSimpleText = nil + published = nil end # TODO: when refactoring video types, make a struct for related videos @@ -55,7 +56,7 @@ def parse_related_video(related : JSON::Any, published : String? = nil) : Hash(S "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), - "publishedText" => JSON::Any.new(publishedSimpleText || ""), + "publishedText" => JSON::Any.new(published_time_text || ""), } end @@ -244,13 +245,7 @@ def parse_video_info(video_id : String, player_response : Hash(String, JSON::Any .dig?("secondaryResults", "secondaryResults", "results") secondary_results.try &.as_a.each do |element| if item = element["compactVideoRenderer"]? - if rv_published_time_text = item["publishedTimeText"]? - rv_decoded_time = decode_date(rv_published_time_text["simpleText"].to_s) - rv_published_timestamp = rv_decoded_time.to_unix.to_s - else - rv_published_timestamp = nil - end - related_video = parse_related_video(item, published: rv_published_timestamp) + related_video = parse_related_video(item) related << JSON::Any.new(related_video) if related_video end end From 7b7197cde8417fd7e43382351da2dc7575c6fc0a Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Mon, 22 Apr 2024 16:26:49 +0300 Subject: [PATCH 21/28] retrigger checks From cab02d49593122fbd157997c61e2a93d2a0e5dc2 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Fri, 16 Aug 2024 13:54:27 +0300 Subject: [PATCH 22/28] Corrected usage of publishedText variable throughout the code --- src/invidious/jsonify/api_v1/video_json.cr | 6 +++++- src/invidious/videos/parser.cr | 1 - 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 622641ee..ea970ea2 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -246,7 +246,11 @@ module Invidious::JSONify::APIv1 json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? - json.field "publishedTimeText", translate(locale, "`x` ago", rv["publishedText"].to_s.gsub(" ago", "")) + if !rv[published].nil? + json.field "publishedText", translate(locale, "`x` ago", recode_date(rv[published], locale)) + else + json.field "publishedText", translate(locale, "`x` ago", "NaN") + end end end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 9275b1bb..3ba57158 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -56,7 +56,6 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? "short_view_count" => JSON::Any.new(short_view_count || "0"), "author_verified" => JSON::Any.new(author_verified), "published" => JSON::Any.new(published || ""), - "publishedText" => JSON::Any.new(published_time_text || ""), } end From 2d6b46c9266f44512de680e161de526487c0ba9e Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Fri, 16 Aug 2024 14:05:13 +0300 Subject: [PATCH 23/28] Fixed a really easy mistake --- src/invidious/jsonify/api_v1/video_json.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index ea970ea2..678ab121 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -246,8 +246,8 @@ module Invidious::JSONify::APIv1 json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? - if !rv[published].nil? - json.field "publishedText", translate(locale, "`x` ago", recode_date(rv[published], locale)) + if !rv["published"].nil? + json.field "publishedText", translate(locale, "`x` ago", recode_date(rv["published"], locale)) else json.field "publishedText", translate(locale, "`x` ago", "NaN") end From 26dc9dc99c51871740d69badf1589975ce7b9589 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Fri, 16 Aug 2024 14:08:04 +0300 Subject: [PATCH 24/28] Solution --- src/invidious/jsonify/api_v1/video_json.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 678ab121..b8fe87c7 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -247,7 +247,7 @@ module Invidious::JSONify::APIv1 json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? if !rv["published"].nil? - json.field "publishedText", translate(locale, "`x` ago", recode_date(rv["published"], locale)) + json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.unix(rv["published"].to_i), locale)) else json.field "publishedText", translate(locale, "`x` ago", "NaN") end From 69ff6def5fd1a5a06acf6d7e074166a7e6f9a6de Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Fri, 16 Aug 2024 14:11:28 +0300 Subject: [PATCH 25/28] Removed useless variable --- src/invidious/videos/parser.cr | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 3ba57158..317a3bf3 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -38,8 +38,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? if published_time_text = related["publishedTimeText"]? decoded_time = decode_date(published_time_text["simpleText"].to_s) - published = decoded_time.to_unix.to_s - published_time_text = published_time_text["simpleText"].to_s + published = decoded_time.to_unix.to_s else published = nil end From e8cd631b2d5c07ab1d83e836838d96f9c25905cb Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Fri, 16 Aug 2024 14:13:05 +0300 Subject: [PATCH 26/28] Formatting --- src/invidious/videos/parser.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 317a3bf3..73f8bbb0 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -38,7 +38,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? if published_time_text = related["publishedTimeText"]? decoded_time = decode_date(published_time_text["simpleText"].to_s) - published = decoded_time.to_unix.to_s + published = decoded_time.to_unix.to_s else published = nil end From b526f481204a4c9324daf9d223a270c36b6d4d60 Mon Sep 17 00:00:00 2001 From: RadoslavL Date: Fri, 16 Aug 2024 23:57:49 +0300 Subject: [PATCH 27/28] Changed Unix time to Rfc3339 time and removed NaN message --- src/invidious/jsonify/api_v1/video_json.cr | 4 ++-- src/invidious/videos/parser.cr | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index b8fe87c7..8c9a5cd3 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -247,9 +247,9 @@ module Invidious::JSONify::APIv1 json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? if !rv["published"].nil? - json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.unix(rv["published"].to_i), locale)) + json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale)) else - json.field "publishedText", translate(locale, "`x` ago", "NaN") + json.field "publishedText", "" end end end diff --git a/src/invidious/videos/parser.cr b/src/invidious/videos/parser.cr index 73f8bbb0..769e3368 100644 --- a/src/invidious/videos/parser.cr +++ b/src/invidious/videos/parser.cr @@ -38,7 +38,7 @@ def parse_related_video(related : JSON::Any) : Hash(String, JSON::Any)? if published_time_text = related["publishedTimeText"]? decoded_time = decode_date(published_time_text["simpleText"].to_s) - published = decoded_time.to_unix.to_s + published = decoded_time.to_rfc3339.to_s else published = nil end From eed14d08a8c33424c2f8d7223351de04b20f47d2 Mon Sep 17 00:00:00 2001 From: RadoslavL <63926126+RadoslavL@users.noreply.github.com> Date: Thu, 31 Oct 2024 09:59:06 +0200 Subject: [PATCH 28/28] Update src/invidious/jsonify/api_v1/video_json.cr Co-authored-by: Samantaz Fox --- src/invidious/jsonify/api_v1/video_json.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/jsonify/api_v1/video_json.cr b/src/invidious/jsonify/api_v1/video_json.cr index 8c9a5cd3..53a4d241 100644 --- a/src/invidious/jsonify/api_v1/video_json.cr +++ b/src/invidious/jsonify/api_v1/video_json.cr @@ -246,7 +246,7 @@ module Invidious::JSONify::APIv1 json.field "viewCountText", rv["short_view_count"]? json.field "viewCount", rv["view_count"]?.try &.empty? ? nil : rv["view_count"].to_i64 json.field "published", rv["published"]? - if !rv["published"].nil? + if !rv["published"]?.nil? json.field "publishedText", translate(locale, "`x` ago", recode_date(Time.parse_rfc3339(rv["published"].to_s), locale)) else json.field "publishedText", ""