From 600da635b78f3cabee327361866f1ff0c78c0438 Mon Sep 17 00:00:00 2001 From: raphj Date: Sun, 2 Apr 2023 23:36:06 +0200 Subject: [PATCH 01/67] Allow browser suggestions for search (#3704) --- src/invidious/views/components/search_box.ecr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/invidious/views/components/search_box.ecr b/src/invidious/views/components/search_box.ecr index 1240e5bd..a03785d1 100644 --- a/src/invidious/views/components/search_box.ecr +++ b/src/invidious/views/components/search_box.ecr @@ -1,6 +1,6 @@
- autofocus<% end %> name="q" placeholder="<%= translate(locale, "search") %>" title="<%= translate(locale, "search") %>" From b2a0e6f1ffe448f8c3f6f943b34c673537210794 Mon Sep 17 00:00:00 2001 From: chunky programmer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sun, 14 May 2023 16:49:49 -0400 Subject: [PATCH 02/67] Parse playlists when searching a channel --- src/invidious/yt_backend/extractors.cr | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/invidious/yt_backend/extractors.cr b/src/invidious/yt_backend/extractors.cr index 8ff4c1f9..6686e6e7 100644 --- a/src/invidious/yt_backend/extractors.cr +++ b/src/invidious/yt_backend/extractors.cr @@ -381,7 +381,7 @@ private module Parsers # Parses an InnerTube itemSectionRenderer into a SearchVideo. # Returns nil when the given object isn't a ItemSectionRenderer # - # A itemSectionRenderer seems to be a simple wrapper for a videoRenderer, used + # A itemSectionRenderer seems to be a simple wrapper for a videoRenderer or a playlistRenderer, used # by the result page for channel searches. It is located inside a continuationItems # container.It is very similar to RichItemRendererParser # @@ -394,6 +394,8 @@ private module Parsers private def self.parse(item_contents, author_fallback) child = VideoRendererParser.process(item_contents, author_fallback) + child ||= PlaylistRendererParser.process(item_contents, author_fallback) + return child end From c713c32cebda5d0199b5c0dd553744f8d61707da Mon Sep 17 00:00:00 2001 From: chunky programmer <78101139+ChunkyProgrammer@users.noreply.github.com> Date: Sun, 14 May 2023 22:35:51 -0400 Subject: [PATCH 03/67] Fix issue where playlists will refetch the same videos --- src/invidious/routes/playlists.cr | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/invidious/routes/playlists.cr b/src/invidious/routes/playlists.cr index 0d242ee6..8675fa45 100644 --- a/src/invidious/routes/playlists.cr +++ b/src/invidious/routes/playlists.cr @@ -410,8 +410,8 @@ module Invidious::Routes::Playlists return error_template(500, ex) end - page_count = (playlist.video_count / 100).to_i - page_count += 1 if (playlist.video_count % 100) > 0 + page_count = (playlist.video_count / 200).to_i + page_count += 1 if (playlist.video_count % 200) > 0 if page > page_count return env.redirect "/playlist?list=#{plid}&page=#{page_count}" @@ -422,7 +422,7 @@ module Invidious::Routes::Playlists end begin - videos = get_playlist_videos(playlist, offset: (page - 1) * 100) + videos = get_playlist_videos(playlist, offset: (page - 1) * 200) rescue ex return error_template(500, "Error encountered while retrieving playlist videos.
#{ex.message}") end From 898066407d85a2844c87fa6fc0e8179977cabb9c Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 29 May 2023 12:41:53 +0200 Subject: [PATCH 04/67] Utils: Update 'decode_date' to take into account short "x ago" forms --- src/invidious/helpers/utils.cr | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/src/invidious/helpers/utils.cr b/src/invidious/helpers/utils.cr index bcf7c963..48bf769f 100644 --- a/src/invidious/helpers/utils.cr +++ b/src/invidious/helpers/utils.cr @@ -111,24 +111,27 @@ def decode_date(string : String) else nil # Continue end - # String matches format "20 hours ago", "4 months ago"... - date = string.split(" ")[-3, 3] - delta = date[0].to_i + # String matches format "20 hours ago", "4 months ago", "20s ago", "15min ago"... + match = string.match(/(?\d+) ?(?[smhdwy]\w*) ago/) - case date[1] - when .includes? "second" + raise "Could not parse #{string}" if match.nil? + + delta = match["count"].to_i + + case match["span"] + when .starts_with? "s" # second(s) delta = delta.seconds - when .includes? "minute" + when .starts_with? "mi" # minute(s) delta = delta.minutes - when .includes? "hour" + when .starts_with? "h" # hour(s) delta = delta.hours - when .includes? "day" + when .starts_with? "d" # day(s) delta = delta.days - when .includes? "week" + when .starts_with? "w" # week(s) delta = delta.weeks - when .includes? "month" + when .starts_with? "mo" # month(s) delta = delta.months - when .includes? "year" + when .starts_with? "y" # year(s) delta = delta.years else raise "Could not parse #{string}" From 4414c9df70580008c8817ace026b765e83c052aa Mon Sep 17 00:00:00 2001 From: Samantaz Fox Date: Mon, 29 May 2023 12:42:19 +0200 Subject: [PATCH 05/67] specc: Add tests for 'decode_date' --- spec/invidious/utils_spec.cr | 46 ++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100644 spec/invidious/utils_spec.cr diff --git a/spec/invidious/utils_spec.cr b/spec/invidious/utils_spec.cr new file mode 100644 index 00000000..7c2c2711 --- /dev/null +++ b/spec/invidious/utils_spec.cr @@ -0,0 +1,46 @@ +require "../spec_helper" + +Spectator.describe "Utils" do + describe "decode_date" do + it "parses short dates (en-US)" do + expect(decode_date("1s ago")).to be_close(Time.utc - 1.second, 500.milliseconds) + expect(decode_date("2min ago")).to be_close(Time.utc - 2.minutes, 500.milliseconds) + expect(decode_date("3h ago")).to be_close(Time.utc - 3.hours, 500.milliseconds) + expect(decode_date("4d ago")).to be_close(Time.utc - 4.days, 500.milliseconds) + expect(decode_date("5w ago")).to be_close(Time.utc - 5.weeks, 500.milliseconds) + expect(decode_date("6mo ago")).to be_close(Time.utc - 6.months, 500.milliseconds) + expect(decode_date("7y ago")).to be_close(Time.utc - 7.years, 500.milliseconds) + end + + it "parses short dates (en-GB)" do + expect(decode_date("55s ago")).to be_close(Time.utc - 55.seconds, 500.milliseconds) + expect(decode_date("44min ago")).to be_close(Time.utc - 44.minutes, 500.milliseconds) + expect(decode_date("22hr ago")).to be_close(Time.utc - 22.hours, 500.milliseconds) + expect(decode_date("1day ago")).to be_close(Time.utc - 1.day, 500.milliseconds) + expect(decode_date("2days ago")).to be_close(Time.utc - 2.days, 500.milliseconds) + expect(decode_date("3wk ago")).to be_close(Time.utc - 3.weeks, 500.milliseconds) + expect(decode_date("11mo ago")).to be_close(Time.utc - 11.months, 500.milliseconds) + expect(decode_date("11yr ago")).to be_close(Time.utc - 11.years, 500.milliseconds) + end + + it "parses long forms (singular)" do + expect(decode_date("1 second ago")).to be_close(Time.utc - 1.second, 500.milliseconds) + expect(decode_date("1 minute ago")).to be_close(Time.utc - 1.minute, 500.milliseconds) + expect(decode_date("1 hour ago")).to be_close(Time.utc - 1.hour, 500.milliseconds) + expect(decode_date("1 day ago")).to be_close(Time.utc - 1.day, 500.milliseconds) + expect(decode_date("1 week ago")).to be_close(Time.utc - 1.week, 500.milliseconds) + expect(decode_date("1 month ago")).to be_close(Time.utc - 1.month, 500.milliseconds) + expect(decode_date("1 year ago")).to be_close(Time.utc - 1.year, 500.milliseconds) + end + + it "parses long forms (plural)" do + expect(decode_date("5 seconds ago")).to be_close(Time.utc - 5.seconds, 500.milliseconds) + expect(decode_date("17 minutes ago")).to be_close(Time.utc - 17.minutes, 500.milliseconds) + expect(decode_date("23 hours ago")).to be_close(Time.utc - 23.hours, 500.milliseconds) + expect(decode_date("3 days ago")).to be_close(Time.utc - 3.days, 500.milliseconds) + expect(decode_date("2 weeks ago")).to be_close(Time.utc - 2.weeks, 500.milliseconds) + expect(decode_date("9 months ago")).to be_close(Time.utc - 9.months, 500.milliseconds) + expect(decode_date("8 years ago")).to be_close(Time.utc - 8.years, 500.milliseconds) + end + end +end From 042ad1f2662503c123ba1dd415e5ed3d9ddc3cc0 Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Sat, 3 Jun 2023 13:06:48 +0200 Subject: [PATCH 06/67] auto close duplicated issues --- .github/workflows/auto-close-duplicate.yaml | 35 +++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 .github/workflows/auto-close-duplicate.yaml diff --git a/.github/workflows/auto-close-duplicate.yaml b/.github/workflows/auto-close-duplicate.yaml new file mode 100644 index 00000000..3e977a84 --- /dev/null +++ b/.github/workflows/auto-close-duplicate.yaml @@ -0,0 +1,35 @@ +name: Close duplicates +on: + issues: + types: [opened] +jobs: + run: + runs-on: ubuntu-latest + permissions: write-all + steps: + - uses: iv-org/close-potential-duplicates@v1 + with: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # Issue title filter work with anymatch https://www.npmjs.com/package/anymatch. + # Any matched issue will stop detection immediately. + # You can specify multi filters in each line. + filter: '' + # Exclude keywords in title before detecting. + exclude: '' + # Label to set, when potential duplicates are detected. + label: duplicate + # Get issues with state to compare. Supported state: 'all', 'closed', 'open'. + state: open + # If similarity is higher than this threshold([0,1]), issue will be marked as duplicate. + threshold: 0.6 + # Reactions to be add to comment when potential duplicates are detected. + # Available reactions: "-1", "+1", "confused", "laugh", "heart", "hooray", "rocket", "eyes" + reactions: '' + close: true + # Comment to post when potential duplicates are detected. + comment: > + Hello, your issue is a duplicate of this/these issue(s): {{#issues}} + - #{{ number }} [accuracy: ({{ accuracy }}%)] + {{/issues}} + If this is a mistake please explain why and ping @\unixfox, @\SamantazFox and @\TheFrenchGhosty. + Please refrain from opening new issues, it won't help in solving your problem. \ No newline at end of file From 7ea6ec1f52ae02cbc35401ad272433d7073d8866 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos=20=28perso=29?= Date: Sat, 3 Jun 2023 18:57:42 +0200 Subject: [PATCH 07/67] add one return line for the reply message --- .github/workflows/auto-close-duplicate.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/auto-close-duplicate.yaml b/.github/workflows/auto-close-duplicate.yaml index 3e977a84..4495c22e 100644 --- a/.github/workflows/auto-close-duplicate.yaml +++ b/.github/workflows/auto-close-duplicate.yaml @@ -32,4 +32,5 @@ jobs: - #{{ number }} [accuracy: ({{ accuracy }}%)] {{/issues}} If this is a mistake please explain why and ping @\unixfox, @\SamantazFox and @\TheFrenchGhosty. + Please refrain from opening new issues, it won't help in solving your problem. \ No newline at end of file From bc06c2fc27a9f1fb4edb8a2af570d67c0af5ba0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=89milien=20Devos=20=28perso=29?= Date: Sat, 3 Jun 2023 17:27:24 +0000 Subject: [PATCH 08/67] Better message for auto close --- .github/workflows/auto-close-duplicate.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/auto-close-duplicate.yaml b/.github/workflows/auto-close-duplicate.yaml index 4495c22e..aa6457ed 100644 --- a/.github/workflows/auto-close-duplicate.yaml +++ b/.github/workflows/auto-close-duplicate.yaml @@ -27,10 +27,11 @@ jobs: reactions: '' close: true # Comment to post when potential duplicates are detected. - comment: > + comment: | Hello, your issue is a duplicate of this/these issue(s): {{#issues}} - - #{{ number }} [accuracy: ({{ accuracy }}%)] + - #{{ number }} [accuracy: {{ accuracy }}%] {{/issues}} + If this is a mistake please explain why and ping @\unixfox, @\SamantazFox and @\TheFrenchGhosty. - Please refrain from opening new issues, it won't help in solving your problem. \ No newline at end of file + Please refrain from opening new issues, it won't help in solving your problem. From 372192eabc9a23373023d0ed9209059138bb4e66 Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Sun, 4 Jun 2023 17:13:48 +0200 Subject: [PATCH 09/67] warn about hmac key deadline --- src/invidious.cr | 9 +++++++-- src/invidious/views/template.ecr | 8 ++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index b5abd5c7..27c4775e 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -57,8 +57,9 @@ end # Simple alias to make code easier to read alias IV = Invidious -CONFIG = Config.load -HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) +CONFIG = Config.load +HMAC_KEY_CONFIGURED = CONFIG.hmac_key != nil +HMAC_KEY = CONFIG.hmac_key || Random::Secure.hex(32) PG_DB = DB.open CONFIG.database_url ARCHIVE_URL = URI.parse("https://archive.org") @@ -230,6 +231,10 @@ Kemal.config.host_binding = Kemal.config.host_binding != "0.0.0.0" ? Kemal.confi Kemal.config.port = Kemal.config.port != 3000 ? Kemal.config.port : CONFIG.port Kemal.config.app_name = "Invidious" +if !HMAC_KEY_CONFIGURED + LOGGER.warn("Please configure hmac_key by July 1st, see more here: https://github.com/iv-org/invidious/issues/3854") +end + # Use in kemal's production mode. # Users can also set the KEMAL_ENV environmental variable for this to be set automatically. {% if flag?(:release) || flag?(:production) %} diff --git a/src/invidious/views/template.ecr b/src/invidious/views/template.ecr index 77265679..aa0fc15f 100644 --- a/src/invidious/views/template.ecr +++ b/src/invidious/views/template.ecr @@ -111,6 +111,14 @@ <% end %> + <% if env.get? "user" %> + <% if !HMAC_KEY_CONFIGURED && CONFIG.admins.includes? env.get("user").as(Invidious::User).email %> +
+

Message for admin: please configure hmac_key, see more here.

+
+ <% end %> + <% end %> + <%= content %>