Fix fallback for comments

This reverts commit 95c6747a3e.

CHANGELOG.md

@@ -1,3 +1,17 @@
+# 0.9.0 (2018-10-08)
+
+## Week 9: Playlists
+
+Not as much to announce this week, but I'm still quite happy to announce a couple things, namely:
+
+Playback support for playlists has finally been added with [`88430a6`](https://github.com/omarroth/invidious/88430a6). You can now view playlists with the `&list=` query param, as you would on YouTube. You can also view mixes with the mentioned `&list=`, although they require some extra handling that I would like to add in the coming week, as well as adding playlist looping and shuffle. I think playback support has been a roadblock for more exciting features such as [#114](https://github.com/omarroth/invidious/issues/114), and I look forward to improving the experience.
+
+Comments have had a bit of a cosmetic upgrade with [#132](https://github.com/omarroth/invidious/issues/132), which I think helps better distinguish between Reddit and YouTube comments, as it makes them appear similarly to their respective sites. You can also now switch between YouTube and Reddit comments with a push of a button, which I think is quite an improvement, especially for newer or less popular videos with fewer comments.
+
+I've had a small breakthrough in speeding up users' subscription feeds with PostgreSQL's [materialized views](https://www.postgresql.org/docs/current/static/rules-materializedviews.html). Without going into too much detail, materialized views essentially cache the result of a query, making it possible to run resource-intensive queries once, rather than every time a user visits their feed. In the coming week I hope to push this out to users, and hopefully close [#173](https://github.com/omarroth/invidious/issues/173).
+
+I haven't had as much time to work on the project this week, but I'm quite happy to have added some new features. Have a great week everyone.
+
 # 0.8.0 (2018-10-02)
 
 ## Week 8: Mixes

assets/css/default.css

@@ -22,6 +22,10 @@ div {
   padding-right: 10px;
 }
 
+.pure-button-primary {
+  background: rgba(0, 182, 240, 1);
+}
+
 /*
  * Navbar
  */

assets/js/watch.js

@@ -21,11 +21,6 @@ function toggle_comments(target) {
 }
 
 function swap_comments(source) {
-  comments = document.getElementById("comments");
-  var fallback = comments.innerHTML;
-  comments.innerHTML =
-    '<h3><center class="loading"><i class="icon ion-ios-refresh"></i></center></h3>';
-
   if (source == "youtube") {
     get_youtube_comments();
   } else if (source == "reddit") {
@@ -46,3 +41,19 @@ String.prototype.supplant = function(o) {
     return typeof r === "string" || typeof r === "number" ? r : a;
   });
 };
+
+function show_youtube_replies(target) {
+  body = target.parentNode.parentNode.children[1];
+  body.style.display = "";
+
+  target.innerHTML = "Hide replies";
+  target.setAttribute("onclick", "hide_youtube_replies(this)");
+}
+
+function hide_youtube_replies(target) {
+  body = target.parentNode.parentNode.children[1];
+  body.style.display = "none";
+
+  target.innerHTML = "Show replies";
+  target.setAttribute("onclick", "show_youtube_replies(this)");
+}

config/config.yml

@@ -1,5 +1,6 @@
 crawl_threads: 1
 channel_threads: 1
+feed_threads: 1
 video_threads: 1
 db:
   user: kemal

config/sql/videos.sql

@@ -22,6 +22,8 @@ CREATE TABLE public.videos
     genre text COLLATE pg_catalog."default",
     genre_url text COLLATE pg_catalog."default",
     license text COLLATE pg_catalog."default",
+    sub_count_text text COLLATE pg_catalog."default",
+    author_thumbnail text COLLATE pg_catalog."default",
     CONSTRAINT videos_pkey PRIMARY KEY (id)
 )
 WITH (

shard.yml

@@ -1,5 +1,5 @@
 name: invidious
-version: 0.8.0
+version: 0.9.0
 
 authors:
   - Omar Roth <omarroth@hotmail.com>

src/invidious.cr

@@ -31,6 +31,7 @@ HMAC_KEY = CONFIG.hmac_key || Random::Secure.random_bytes(32)
 
 crawl_threads = CONFIG.crawl_threads
 channel_threads = CONFIG.channel_threads
+feed_threads = CONFIG.feed_threads
 video_threads = CONFIG.video_threads
 
 Kemal.config.extra_options do |parser|
@@ -51,6 +52,14 @@ Kemal.config.extra_options do |parser|
       exit
     end
   end
+  parser.on("-f THREADS", "--feed-threads=THREADS", "Number of threads for refreshing feeds (default: #{feed_threads})") do |number|
+    begin
+      feed_threads = number.to_i
+    rescue ex
+      puts "THREADS must be integer"
+      exit
+    end
+  end
   parser.on("-v THREADS", "--video-threads=THREADS", "Number of threads for refreshing videos (default: #{video_threads})") do |number|
     begin
       video_threads = number.to_i
@@ -85,6 +94,8 @@ end
 
 refresh_channels(PG_DB, channel_threads, CONFIG.full_refresh)
 
+refresh_feeds(PG_DB, feed_threads)
+
 video_threads.times do |i|
   spawn do
     refresh_videos(PG_DB)
@@ -475,9 +486,8 @@ get "/search" do |env|
   user = env.get? "user"
   if user
     user = user.as(User)
-    ucids = user.subscriptions
+    view_name = "subscriptions_#{sha256(user.email)[0..7]}"
   end
-  ucids ||= [] of String
 
   channel = nil
   content_type = "all"
@@ -514,14 +524,19 @@ get "/search" do |env|
   if channel
     count, videos = channel_search(search_query, page, channel)
   elsif subscriptions
-    videos = PG_DB.query_all("SELECT id,title,published,updated,ucid,author FROM (
+    if view_name
+      videos = PG_DB.query_all("SELECT id,title,published,updated,ucid,author FROM (
       SELECT *,
-      to_tsvector(channel_videos.title) ||
-      to_tsvector(channel_videos.author)
+      to_tsvector(#{view_name}.title) ||
+      to_tsvector(#{view_name}.author)
       as document
-      FROM channel_videos WHERE ucid IN (#{arg_array(ucids, 3)})
-      ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", [search_query, (page - 1) * 20] + ucids, as: ChannelVideo)
-    count = videos.size
+      FROM #{view_name}
+      ) v_search WHERE v_search.document @@ plainto_tsquery($1) LIMIT 20 OFFSET $2;", search_query, (page - 1) * 20, as: ChannelVideo)
+      count = videos.size
+    else
+      videos = [] of ChannelVideo
+      count = 0
+    end
   else
     begin
       search_params = produce_search_params(sort: sort, date: date, content_type: content_type,
@@ -755,7 +770,7 @@ post "/login" do |env|
     end
 
     if action == "signin"
-      user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1 AND password IS NOT NULL", email, as: User)
+      user = PG_DB.query_one?("SELECT * FROM users WHERE LOWER(email) = LOWER($1) AND password IS NOT NULL", email, as: User)
 
       if !user
         error_message = "Invalid username or password"
@@ -769,7 +784,7 @@ post "/login" do |env|
 
       if Crypto::Bcrypt::Password.new(user.password.not_nil!) == password
         sid = Base64.urlsafe_encode(Random::Secure.random_bytes(32))
-        PG_DB.exec("UPDATE users SET id = id || $1 WHERE email = $2", [sid], email)
+        PG_DB.exec("UPDATE users SET id = id || $1 WHERE LOWER(email) = LOWER($2)", [sid], email)
 
         if Kemal.config.ssl || CONFIG.https_only
           secure = true
@@ -784,7 +799,7 @@ post "/login" do |env|
         next templated "error"
       end
     elsif action == "register"
-      user = PG_DB.query_one?("SELECT * FROM users WHERE email = $1 AND password IS NOT NULL", email, as: User)
+      user = PG_DB.query_one?("SELECT * FROM users WHERE LOWER(email) = LOWER($1) AND password IS NOT NULL", email, as: User)
       if user
         error_message = "Please sign in"
         next templated "error"
@@ -799,6 +814,12 @@ post "/login" do |env|
 
       PG_DB.exec("INSERT INTO users VALUES (#{args})", user_array)
 
+      view_name = "subscriptions_#{sha256(user.email)[0..7]}"
+      PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS \
+        SELECT * FROM channel_videos WHERE \
+        ucid = ANY ((SELECT subscriptions FROM users WHERE email = '#{user.email}')::text[]) \
+      ORDER BY published DESC;")
+
       if Kemal.config.ssl || CONFIG.https_only
         secure = true
       else
@@ -1364,6 +1385,8 @@ get "/feed/subscriptions" do |env|
 
     notifications = PG_DB.query_one("SELECT notifications FROM users WHERE email = $1", user.email,
       as: Array(String))
+    view_name = "subscriptions_#{sha256(user.email)[0..7]}"
+
     if preferences.notifications_only && !notifications.empty?
       args = arg_array(notifications)
 
@@ -1386,39 +1409,35 @@ get "/feed/subscriptions" do |env|
     else
       if preferences.latest_only
         if preferences.unseen_only
-          ucids = arg_array(user.subscriptions)
           if user.watched.empty?
             watched = "'{}'"
           else
-            watched = arg_array(user.watched, user.subscriptions.size + 1)
+            watched = arg_array(user.watched)
           end
 
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE \
-            ucid IN (#{ucids}) AND id NOT IN (#{watched}) ORDER BY ucid, published DESC",
-            user.subscriptions + user.watched, as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} WHERE \
+            id NOT IN (#{watched}) ORDER BY ucid, published DESC",
+            user.watched, as: ChannelVideo)
         else
-          args = arg_array(user.subscriptions)
-          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE \
-          ucid IN (#{args}) ORDER BY ucid, published DESC", user.subscriptions, as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} \
+          ORDER BY ucid, published DESC", as: ChannelVideo)
         end
 
         videos.sort_by! { |video| video.published }.reverse!
       else
         if preferences.unseen_only
-          ucids = arg_array(user.subscriptions, 3)
           if user.watched.empty?
             watched = "'{}'"
           else
-            watched = arg_array(user.watched, user.subscriptions.size + 3)
+            watched = arg_array(user.watched, 3)
           end
 
-          videos = PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid IN (#{ucids}) \
-          AND id NOT IN (#{watched}) ORDER BY published DESC LIMIT $1 OFFSET $2",
-            [limit, offset] + user.subscriptions + user.watched, as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT * FROM #{view_name} WHERE \
+          id NOT IN (#{watched}) LIMIT $1 OFFSET $2",
+            [limit, offset] + user.watched, as: ChannelVideo)
         else
-          args = arg_array(user.subscriptions, 3)
-          videos = PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid IN (#{args}) \
-          ORDER BY published DESC LIMIT $1 OFFSET $2", [limit, offset] + user.subscriptions, as: ChannelVideo)
+          videos = PG_DB.query_all("SELECT * FROM #{view_name} \
+          ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
         end
       end
 
@@ -1467,29 +1486,8 @@ get "/feed/channel/:ucid" do |env|
     halt env, status_code: 404, response: error_message
   end
 
-  client = make_client(YT_URL)
-
   page = 1
-
-  videos = [] of SearchVideo
-  2.times do |i|
-    url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated)
-    response = client.get(url)
-    json = JSON.parse(response.body)
-
-    if json["content_html"]? && !json["content_html"].as_s.empty?
-      document = XML.parse_html(json["content_html"].as_s)
-      nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
-      if auto_generated
-        videos += extract_videos(nodeset)
-      else
-        videos += extract_videos(nodeset, ucid)
-      end
-    else
-      break
-    end
-  end
+  videos, count = get_60_videos(ucid, page, auto_generated)
 
   host_url = make_host_url(Kemal.config.ssl || CONFIG.https_only, env.request.headers["Host"]?)
   path = env.request.path
@@ -1576,15 +1574,14 @@ get "/feed/private" do |env|
   latest_only ||= 0
   latest_only = latest_only == 1
 
+  view_name = "subscriptions_#{sha256(user.email)[0..7]}"
+
   if latest_only
-    args = arg_array(user.subscriptions)
-    videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM channel_videos WHERE \
-    ucid IN (#{args}) ORDER BY ucid, published DESC", user.subscriptions, as: ChannelVideo)
+    videos = PG_DB.query_all("SELECT DISTINCT ON (ucid) * FROM #{view_name} ORDER BY ucid, published DESC", as: ChannelVideo)
     videos.sort_by! { |video| video.published }.reverse!
   else
-    args = arg_array(user.subscriptions, 3)
-    videos = PG_DB.query_all("SELECT * FROM channel_videos WHERE ucid IN (#{args}) \
-  ORDER BY published DESC LIMIT $1 OFFSET $2", [limit, offset] + user.subscriptions, as: ChannelVideo)
+    videos = PG_DB.query_all("SELECT * FROM #{view_name} \
+    ORDER BY published DESC LIMIT $1 OFFSET $2", limit, offset, as: ChannelVideo)
   end
 
   sort = env.params.query["sort"]?
@@ -1721,7 +1718,7 @@ get "/channel/:ucid" do |env|
   page ||= 1
 
   begin
-    author, ucid, auto_generated = get_about_info(ucid)
+    author, ucid, auto_generated, sub_count = get_about_info(ucid)
   rescue ex
     error_message = "User does not exist"
     next templated "error"
@@ -1735,27 +1732,7 @@ get "/channel/:ucid" do |env|
     end
   end
 
-  client = make_client(YT_URL)
-
-  videos = [] of SearchVideo
-  2.times do |i|
-    url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated)
-    response = client.get(url)
-    json = JSON.parse(response.body)
-
-    if json["content_html"]? && !json["content_html"].as_s.empty?
-      document = XML.parse_html(json["content_html"].as_s)
-      nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
-      if auto_generated
-        videos += extract_videos(nodeset)
-      else
-        videos += extract_videos(nodeset, ucid)
-      end
-    else
-      break
-    end
-  end
+  videos, count = get_60_videos(ucid, page, auto_generated)
 
   templated "channel"
 end
@@ -1908,7 +1885,6 @@ get "/api/v1/comments/:id" do |env|
               proxy_client.read_timeout = 10.seconds
               proxy_client.connect_timeout = 10.seconds
 
-              proxy = list.sample(1)[0]
               proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port])
               proxy_client.set_proxy(proxy)
 
@@ -1917,13 +1893,10 @@ get "/api/v1/comments/:id" do |env|
               proxy_headers["cookie"] = response.cookies.add_request_headers(headers)["cookie"]
               proxy_html = response.body
 
-              if proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
-                bypass_channel.send(nil)
-              else
+              if !proxy_html.match(/<meta itemprop="regionsAllowed" content="">/)
                 bypass_channel.send({proxy_html, proxy_client, proxy_headers})
+                break
               end
-
-              break
             rescue ex
             end
           end
@@ -2281,6 +2254,22 @@ get "/api/v1/videos/:id" do |env|
       json.field "authorId", video.ucid
       json.field "authorUrl", "/channel/#{video.ucid}"
 
+      json.field "authorThumbnails" do
+        json.array do
+          qualities = [32, 48, 76, 100, 176, 512]
+
+          qualities.each do |quality|
+            json.object do
+              json.field "url", video.author_thumbnail.gsub("=s48-", "=s#{quality}-")
+              json.field "width", quality
+              json.field "height", quality
+            end
+          end
+        end
+      end
+
+      json.field "subCountText", video.sub_count_text
+
       json.field "lengthSeconds", video.info["length_seconds"].to_i
       if video.info["allow_ratings"]?
         json.field "allowRatings", video.info["allow_ratings"] == "1"
@@ -2499,30 +2488,10 @@ get "/api/v1/channels/:ucid" do |env|
     halt env, status_code: 404, response: error_message
   end
 
-  client = make_client(YT_URL)
-
   page = 1
+  videos, count = get_60_videos(ucid, page, auto_generated)
 
-  videos = [] of SearchVideo
-  2.times do |i|
-    url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated)
-    response = client.get(url)
-    json = JSON.parse(response.body)
-
-    if json["content_html"]? && !json["content_html"].as_s.empty?
-      document = XML.parse_html(json["content_html"].as_s)
-      nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
-      if auto_generated
-        videos += extract_videos(nodeset)
-      else
-        videos += extract_videos(nodeset, ucid)
-      end
-    else
-      break
-    end
-  end
-
+  client = make_client(YT_URL)
   channel_html = client.get("/channel/#{ucid}/about?disable_polymer=1").body
   channel_html = XML.parse_html(channel_html)
   banner = channel_html.xpath_node(%q(//div[@id="gh-banner"]/style)).not_nil!.content
@@ -2658,27 +2627,7 @@ end
       halt env, status_code: 404, response: error_message
     end
 
-    client = make_client(YT_URL)
-
-    videos = [] of SearchVideo
-    2.times do |i|
-      url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated)
-      response = client.get(url)
-      json = JSON.parse(response.body)
-
-      if json["content_html"]? && !json["content_html"].as_s.empty?
-        document = XML.parse_html(json["content_html"].as_s)
-        nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
-
-        if auto_generated
-          videos += extract_videos(nodeset)
-        else
-          videos += extract_videos(nodeset, ucid)
-        end
-      else
-        break
-      end
-    end
+    videos, count = get_60_videos(ucid, page, auto_generated)
 
     result = JSON.build do |json|
       json.array do

src/invidious/channels.cr

@@ -176,7 +176,7 @@ def produce_channel_videos_url(ucid, page = 1, auto_generated = nil)
   continuation = Base64.urlsafe_encode(continuation)
   continuation = URI.escape(continuation)
 
-  url = "/browse_ajax?continuation=#{continuation}"
+  url = "/browse_ajax?continuation=#{continuation}&gl=US&hl=en"
 
   return url
 end
@@ -196,6 +196,12 @@ def get_about_info(ucid)
     raise "User does not exist."
   end
 
+  sub_count = about.xpath_node(%q(//span[contains(text(), "subscribers")]))
+  if sub_count
+    sub_count = sub_count.content.delete(", subscribers").to_i?
+  end
+  sub_count ||= 0
+
   author = about.xpath_node(%q(//span[@class="qualified-channel-title-text"]/a)).not_nil!.content
   ucid = about.xpath_node(%q(//link[@rel="canonical"])).not_nil!["href"].split("/")[-1]
 
@@ -207,5 +213,37 @@ def get_about_info(ucid)
     auto_generated = true
   end
 
-  return {author, ucid, auto_generated}
+  return {author, ucid, auto_generated, sub_count}
+end
+
+def get_60_videos(ucid, page, auto_generated)
+  count = 0
+  videos = [] of SearchVideo
+
+  client = make_client(YT_URL)
+
+  2.times do |i|
+    url = produce_channel_videos_url(ucid, page * 2 + (i - 1), auto_generated: auto_generated)
+    response = client.get(url)
+    json = JSON.parse(response.body)
+
+    if json["content_html"]? && !json["content_html"].as_s.empty?
+      document = XML.parse_html(json["content_html"].as_s)
+      nodeset = document.xpath_nodes(%q(//li[contains(@class, "feed-item-container")]))
+
+      if !json["load_more_widget_html"]?.try &.as_s.empty?
+        count += 30
+      end
+
+      if auto_generated
+        videos += extract_videos(nodeset)
+      else
+        videos += extract_videos(nodeset, ucid)
+      end
+    else
+      break
+    end
+  end
+
+  return videos, count
 end

src/invidious/comments.cr

@@ -109,11 +109,9 @@ def template_youtube_comments(comments)
       </div>
       <div class="pure-u-20-24 pure-u-md-22-24">
         <p>
-          <a href="javascript:void(0)" onclick="toggle_parent(this)">[ - ]</a> 
           <b>
             <a href="#{child["authorUrl"]}">#{child["author"]}</a>
           </b> 
-          <div>
           <p style="white-space:pre-wrap">#{child["contentHtml"]}</p>
           #{recode_date(Time.epoch(child["published"].as_i64))} ago
           | 

src/invidious/helpers/helpers.cr

@@ -2,6 +2,7 @@ class Config
   YAML.mapping({
     crawl_threads:   Int32,
     channel_threads: Int32,
+    feed_threads:    Int32,
     video_threads:   Int32,
     db:              NamedTuple(
       user: String,

src/invidious/helpers/utils.cr

@@ -238,3 +238,9 @@ def write_var_int(value : Int)
 
   return bytes
 end
+
+def sha256(text)
+  digest = OpenSSL::Digest.new("SHA256")
+  digest << text
+  return digest.hexdigest
+end

src/invidious/jobs.cr

@@ -104,6 +104,44 @@ def refresh_videos(db)
   end
 end
 
+def refresh_feeds(db, max_threads = 1)
+  max_channel = Channel(Int32).new
+
+  spawn do
+    max_threads = max_channel.receive
+    active_threads = 0
+    active_channel = Channel(Bool).new
+
+    loop do
+      db.query("SELECT email FROM users") do |rs|
+        rs.each do
+          email = rs.read(String)
+          view_name = "subscriptions_#{sha256(email)[0..7]}"
+
+          if active_threads >= max_threads
+            if active_channel.receive
+              active_threads -= 1
+            end
+          end
+
+          active_threads += 1
+          spawn do
+            begin
+              db.exec("REFRESH MATERIALIZED VIEW #{view_name}")
+            rescue ex
+              STDOUT << "REFRESH " << email << " : " << ex.message << "\n"
+            end
+
+            active_channel.send(true)
+          end
+        end
+      end
+    end
+  end
+
+  max_channel.send(max_threads)
+end
+
 def pull_top_videos(config, db)
   if config.dl_api_key
     DetectLanguage.configure do |dl_config|

src/invidious/playlists.cr

@@ -65,6 +65,11 @@ def fetch_playlist_videos(plid, page, video_count, continuation = nil)
       nodeset = document.xpath_nodes(%q(.//tr[contains(@class, "pl-video")]))
 
       videos = extract_playlist(plid, nodeset, 0)
+      if continuation
+        until videos[0].id == continuation
+          videos.shift
+        end
+      end
     end
   end
 

src/invidious/users.cr

@@ -119,6 +119,15 @@ def get_user(sid, client, headers, db, refresh = true)
 
       db.exec("INSERT INTO users VALUES (#{args}) \
       ON CONFLICT (email) DO UPDATE SET id = users.id || $1, updated = $2, subscriptions = $4", user_array)
+
+      begin
+        view_name = "subscriptions_#{sha256(user.email)[0..7]}"
+        PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS \
+        SELECT * FROM channel_videos WHERE \
+        ucid = ANY ((SELECT subscriptions FROM users WHERE email = '#{user.email}')::text[]) \
+        ORDER BY published DESC;")
+      rescue ex
+      end
     end
   else
     user = fetch_user(sid, client, headers, db)
@@ -129,6 +138,15 @@ def get_user(sid, client, headers, db, refresh = true)
 
     db.exec("INSERT INTO users VALUES (#{args}) \
     ON CONFLICT (email) DO UPDATE SET id = users.id || $1, updated = $2, subscriptions = $4", user_array)
+
+    begin
+      view_name = "subscriptions_#{sha256(user.email)[0..7]}"
+      PG_DB.exec("CREATE MATERIALIZED VIEW #{view_name} AS \
+      SELECT * FROM channel_videos WHERE \
+      ucid = ANY ((SELECT subscriptions FROM users WHERE email = '#{user.email}')::text[]) \
+      ORDER BY published DESC;")
+    rescue ex
+    end
   end
 
   return user

src/invidious/videos.cr

@@ -456,7 +456,9 @@ class Video
     is_family_friendly: Bool,
     genre:              String,
     genre_url:          String,
-    license:            {
+    license:            String,
+    sub_count_text:     String,
+    author_thumbnail:   {
       type:    String,
       default: "",
     },
@@ -493,8 +495,8 @@ def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32})
         args = arg_array(video_array[1..-1], 2)
 
         db.exec("UPDATE videos SET (info,updated,title,views,likes,dislikes,wilson_score,\
-          published,description,language,author,ucid, allowed_regions, is_family_friendly,\
-          genre, genre_url, license)\
+          published,description,language,author,ucid,allowed_regions,is_family_friendly,\
+          genre,genre_url,license,sub_count_text,author_thumbnail)\
           = (#{args}) WHERE id = $1", video_array)
       rescue ex
         db.exec("DELETE FROM videos * WHERE id = $1", id)
@@ -571,11 +573,8 @@ def fetch_video(id, proxies)
             info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body)
             if !info["reason"]?
               bypass_channel.send(proxy)
-            else
-              bypass_channel.send(nil)
+              break
             end
-
-            break
           rescue ex
           end
         end
@@ -662,11 +661,25 @@ def fetch_video(id, proxies)
   if license
     license = license.content
   else
-    license ||= ""
+    license = ""
+  end
+
+  sub_count_text = html.xpath_node(%q(//span[contains(@class, "yt-subscriber-count")]))
+  if sub_count_text
+    sub_count_text = sub_count_text["title"]
+  else
+    sub_count_text = "0"
+  end
+
+  author_thumbnail = html.xpath_node(%(//img[@alt="#{author}"]))
+  if author_thumbnail
+    author_thumbnail = author_thumbnail["data-thumb"]
+  else
+    author_thumbnail = ""
   end
 
   video = Video.new(id, info, Time.now, title, views, likes, dislikes, wilson_score, published, description,
-    nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license)
+    nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license, sub_count_text, author_thumbnail)
 
   return video
 end

src/invidious/views/channel.ecr

@@ -13,23 +13,32 @@
     </div>
 </div>
 
-<p class="h-box">
+<div class="h-box">
 <% if user %>
     <% if subscriptions.includes? ucid %>
-    <a href="/subscription_ajax?action_remove_subscriptions=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>">
-        <b>Unsubscribe from <%= author %></b>
-    </a>
+    <p>
+        <a id="subscribe" onclick="unsubscribe()" class="pure-button pure-button-primary" 
+            href="/subscription_ajax?action_remove_subscriptions=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>">
+            <b>Unsubscribe from <%= author %> <%= number_with_separator(sub_count) %></b>
+        </a>
+    </p>
     <% else %>
-    <a href="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>">
-        <b>Subscribe to <%= author %></b>
-    </a>
+    <p>
+        <a id="subscribe" onclick="subscribe()" class="pure-button pure-button-primary" 
+            href="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>">
+            <b>Subscribe to <%= author %> <%= number_with_separator(sub_count) %></b>
+        </a>
+    </p>
     <% end %>
 <% else %>
-    <a href="/login?referer=<%= env.get("current_page") %>">
-        <b>Login to subscribe to <%= author %></b>
-    </a>
+    <p>
+        <a id="subscribe" class="pure-button pure-button-primary" 
+            href="/login?referer=<%= env.get("current_page") %>">
+            <b>Login to subscribe to <%= author %></b>
+        </a>
+    </p>
 <% end %>
-</p>
+</div>
 
 <p class="h-box">
     <a href="https://www.youtube.com/channel/<%= ucid %>">View channel on YouTube</a>
@@ -51,8 +60,50 @@
     </div>
     <div class="pure-u-1 pure-u-md-3-5"></div>
     <div style="text-align:right;" class="pure-u-1 pure-u-md-1-5">
-    <% if videos.size == 60 %>
+    <% if count == 60 %>
         <a href="/channel/<%= ucid %>?page=<%= page + 1 %>">Next page</a>
     <% end %>
     </div>
 </div>
+
+<script>
+document.getElementById("subscribe")["href"] = "javascript:void(0);"
+
+function subscribe() {
+    var url = "/subscription_ajax?action_create_subscription_to_channel=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>";
+    var xhr = new XMLHttpRequest();
+    xhr.responseType = "json";
+    xhr.timeout = 20000;
+    xhr.open("GET", url, true);
+    xhr.send();
+
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState == 4) {
+            if (xhr.status == 200) {
+                subscribe_button = document.getElementById("subscribe");
+                subscribe_button.onclick = unsubscribe;
+                subscribe_button.innerHTML = '<b>Unsubscribe from <%= author %> <%= number_with_separator(sub_count + 1) %></b>'
+            }
+        }
+    }
+}
+
+function unsubscribe() {
+    var url = "/subscription_ajax?action_remove_subscriptions=1&c=<%= ucid %>&referer=<%= env.get("current_page") %>";
+    var xhr = new XMLHttpRequest();
+    xhr.responseType = "json";
+    xhr.timeout = 20000;
+    xhr.open("GET", url, true);
+    xhr.send();
+
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState == 4) {
+            if (xhr.status == 200) {
+                subscribe_button = document.getElementById("subscribe");
+                subscribe_button.onclick = subscribe;
+                subscribe_button.innerHTML = '<b>Subscribe to <%= author %> <%= number_with_separator(sub_count) %></b>'
+            }
+        }
+    }
+}
+</script>

src/invidious/views/watch.ecr

@@ -92,20 +92,23 @@
         <% if user %>
             <% if subscriptions.includes? video.ucid %>
             <p>
-                <a href="/subscription_ajax?action_remove_subscriptions=1&c=<%= video.ucid %>&referer=<%= env.get("current_page") %>">
-                    <b>Unsubscribe from <%= video.author %></b>
+                <a id="subscribe" onclick="unsubscribe()" class="pure-button pure-button-primary" 
+                    href="/subscription_ajax?action_remove_subscriptions=1&c=<%= video.ucid %>&referer=<%= env.get("current_page") %>">
+                    <b>Unsubscribe from <%= video.author %> <%= video.sub_count_text %></b>
                 </a>
             </p>
             <% else %>
             <p>
-                <a href="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= video.ucid %>&referer=<%= env.get("current_page") %>">
-                    <b>Subscribe to <%= video.author %></b>
+                <a id="subscribe" onclick="subscribe()" class="pure-button pure-button-primary" 
+                    href="/subscription_ajax?action_create_subscription_to_channel=1&c=<%= video.ucid %>&referer=<%= env.get("current_page") %>">
+                    <b>Subscribe to <%= video.author %> <%= video.sub_count_text %></b>
                 </a>
             </p>
             <% end %>
         <% else %>
             <p>
-                <a href="/login?referer=<%= env.get("current_page") %>">
+                <a id="subscribe" class="pure-button pure-button-primary" 
+                    href="/login?referer=<%= env.get("current_page") %>">
                     <b>Login to subscribe to <%= video.author %></b>
                 </a>
             </p>
@@ -118,15 +121,12 @@
             </div>
             <hr>
             <div id="comments">
-                <h3><center class="loading"><i class="icon ion-ios-refresh"></i></center></h3>
             </div>
         </div>
     </div>
     <div class="pure-u-1 pure-u-md-1-5">
         <% if plid %>
         <div id="playlist" class="h-box">
-            <h3><center class="loading"><i class="icon ion-ios-refresh"></i></center></h3>
-            <hr>
         </div>
         <% end %>
 
@@ -152,8 +152,56 @@
 </div>
 
 <script>
+subscribe_button = document.getElementById("subscribe");
+if (subscribe_button.getAttribute('onclick')) {
+    subscribe_button["href"] = "javascript:void(0);";
+}
+
+function subscribe() {
+    var url = "/subscription_ajax?action_create_subscription_to_channel=1&c=<%= video.ucid %>&referer=<%= env.get("current_page") %>";
+    var xhr = new XMLHttpRequest();
+    xhr.responseType = "json";
+    xhr.timeout = 20000;
+    xhr.open("GET", url, true);
+    xhr.send();
+
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState == 4) {
+            if (xhr.status == 200) {
+                subscribe_button = document.getElementById("subscribe");
+                subscribe_button.onclick = unsubscribe;
+                subscribe_button.innerHTML = '<b>Unsubscribe from <%= video.author %> <%= video.sub_count_text %></b>'
+            }
+        }
+    }
+}
+
+function unsubscribe() {
+    var url = "/subscription_ajax?action_remove_subscriptions=1&c=<%= video.ucid %>&referer=<%= env.get("current_page") %>";
+    var xhr = new XMLHttpRequest();
+    xhr.responseType = "json";
+    xhr.timeout = 20000;
+    xhr.open("GET", url, true);
+    xhr.send();
+
+    xhr.onreadystatechange = function() {
+        if (xhr.readyState == 4) {
+            if (xhr.status == 200) {
+                subscribe_button = document.getElementById("subscribe");
+                subscribe_button.onclick = subscribe;
+                subscribe_button.innerHTML = '<b>Subscribe to <%= video.author %> <%= video.sub_count_text %></b>'
+            }
+        }
+    }
+}
+
 <% if plid %>
 function get_playlist() {
+    playlist = document.getElementById("playlist");
+    playlist.innerHTML = ' \
+      <h3><center class="loading"><i class="icon ion-ios-refresh"></i></center></h3> \
+      <hr>'
+
     var plid = "<%= plid %>"
 
     if (plid.startsWith("RD")) {
@@ -171,7 +219,6 @@ function get_playlist() {
     xhr.onreadystatechange = function() {
         if (xhr.readyState == 4) {
             if (xhr.status == 200) {
-                playlist = document.getElementById("playlist");
                 playlist.innerHTML = xhr.response.playlistHtml;
                 
                 if (xhr.response.nextVideo) {
@@ -185,6 +232,9 @@ function get_playlist() {
                             <% if params[:autoplay] %>
                             + "&autoplay=1"
                             <% end %>
+                            <% if params[:speed] %>
+                            + "&speed=<%= params[:speed] %>"
+                            <% end %>
                         );
                     });
                 }
@@ -208,6 +258,11 @@ get_playlist();
 <% end %>
 
 function get_reddit_comments() {
+  comments = document.getElementById("comments");
+  var fallback = comments.innerHTML;
+  comments.innerHTML =
+    '<h3><center class="loading"><i class="icon ion-ios-refresh"></i></center></h3>';
+
   var url = "/api/v1/comments/<%= video.id %>?source=reddit&format=html";
   var xhr = new XMLHttpRequest();
   xhr.responseType = "json";
@@ -218,7 +273,6 @@ function get_reddit_comments() {
   xhr.onreadystatechange = function() {
     if (xhr.readyState == 4) {
       if (xhr.status == 200) {
-        comments = document.getElementById("comments");
         comments.innerHTML = ' \
             <div> \
                 <h3> \
@@ -246,8 +300,7 @@ function get_reddit_comments() {
         <% if preferences && preferences.comments[1] == "youtube" %> 
         get_youtube_comments();
         <% else %>
-        comments = document.getElementById("comments");
-        comments.innerHTML = "";
+        comments.innerHTML = fallback;
         <% end %>
       }
     }
@@ -261,6 +314,11 @@ function get_reddit_comments() {
 }
 
 function get_youtube_comments() {
+  comments = document.getElementById("comments");
+  var fallback = comments.innerHTML;
+  comments.innerHTML =
+    '<h3><center class="loading"><i class="icon ion-ios-refresh"></i></center></h3>';
+
   var url = "/api/v1/comments/<%= video.id %>?format=html";
   var xhr = new XMLHttpRequest();
   xhr.responseType = "json";
@@ -271,7 +329,6 @@ function get_youtube_comments() {
   xhr.onreadystatechange = function() {
     if (xhr.readyState == 4) {
       if (xhr.status == 200) {
-        comments = document.getElementById("comments");
         if (xhr.response.commentCount > 0)  {
             comments.innerHTML = ' \
                 <div> \
@@ -297,7 +354,6 @@ function get_youtube_comments() {
         <% if preferences && preferences.comments[1] == "youtube" %> 
         get_youtube_comments();
         <% else %>
-        comments = document.getElementById("comments");
         comments.innerHTML = "";
         <% end %>
       }
@@ -307,7 +363,6 @@ function get_youtube_comments() {
   xhr.ontimeout = function() {
     console.log("Pulling comments timed out.");
 
-    comments = document.getElementById("comments");
     comments.innerHTML =
       '<h3><center class="loading"><i class="icon ion-ios-refresh"></i></center></h3>';
     get_youtube_comments();
@@ -333,7 +388,13 @@ function get_youtube_replies(target) {
   xhr.onreadystatechange = function() {
     if (xhr.readyState == 4) {
       if (xhr.status == 200) {
-        body.innerHTML = xhr.response.contentHtml;
+        body.innerHTML = ' \
+            <p><a href="javascript:void(0)" \
+                onclick="hide_youtube_replies(this)">Hide replies \
+            </a></p> \
+            <div>{contentHtml}</div>'.supplant({
+            contentHtml: xhr.response.contentHtml,
+        });
       } else {
         body.innerHTML = fallback;
       }