Add engagement rating, rating, likes, dislikes, and form of logging (primitive but in the hope of reverse-engineering requests)

shard.lock

@@ -6,7 +6,7 @@ shards:
 
   kemal:
     github: kemalcr/kemal
-    version: 0.21.0
+    commit: 8cb9770ec3c6cf5897e644229dad8d0b5c360941
 
   kilt:
     github: jeromegn/kilt

shard.yml

@@ -11,6 +11,7 @@ targets:
 dependencies:
   kemal:
      github: kemalcr/kemal
+     branch: master
   pg:
     github: will/crystal-pg
 

src/visor.cr

@@ -3,93 +3,63 @@ require "json"
 require "kemal"
 require "pg"
 require "xml"
+require "./url_encoded"
 
 macro templated(filename)
   render "views/#{{{filename}}}.ecr", "views/layout.ecr"
 end
 
-# pg = DB.open("postgres://kemal@visor/dev")
-
-alias Type = String | Hash(String, Type)
-
-def object_to_hash(value)
-  object = {} of String => Type
-  items = value.split("&")
-  items.each do |item|
-    key, value = item.split("=")
-    value = URI.unescape(value)
-    object[key] = parse_uri(value)
-  end
-  return object
-end
-
-def array_to_hash(value)
-  array = {} of String => Type
-  items = value.split(",")
-  count = 0
-  items.each do |item|
-    array[count.to_s] = parse_uri(item)
-    count += 1
-  end
-  return array
-end
-
-def parse_uri(value)
-  if value.starts_with?("http") || value.starts_with?("[")
-    return value
-  else
-    if value.includes?(",")
-      return array_to_hash(value)
-    elsif value.includes?("&")
-      return object_to_hash(value)
-    else
-      return value
-    end
-  end
-end
-
 context = OpenSSL::SSL::Context::Client.insecure
-client = HTTP::Client.new("www.youtube.com", 443, context)
+fmt_file = File.open("temp/fmt_stream")
 
 get "/" do |env|
   templated "index"
 end
 
+
 get "/watch/:video_id" do |env|
   video_id = env.params.url["video_id"]
 
-  video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body
+  client = HTTP::Client.new("www.youtube.com", 443, context)
-  video_info = object_to_hash(video_info_encoded)
+  video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
-  body = client.get("/watch?v=#{video_id}").body
+  video_info = HTTP::Params.parse(video_info)
-  doc = XML.parse(body)
+  pageContent = client.get("/watch?v=#{video_id}").body
+  doc = XML.parse(pageContent)
 
+  fmt_stream = [] of HTTP::Params
+  video_info["url_encoded_fmt_stream_map"].split(",") do |string|
+    fmt_stream << HTTP::Params.parse(string)
+  end
+
+  File.write("temp/#{video_id}", video_info)
+  File.write("temp/#{video_id}_manifest", video_info["dashmpd"])
+  File.open("temp/#{video_id}_fmt_stream_0", "a+").puts fmt_stream[0]["url"]
+  File.open("temp/#{video_id}_fmt_stream_1", "a+").puts fmt_stream[1]["url"]
+  File.open("temp/#{video_id}_fmt_stream_2", "a+").puts fmt_stream[2]["url"]
+  File.open("temp/#{video_id}_fmt_stream_3", "a+").puts fmt_stream[3]["url"]
+  fmt_stream.reverse! # We want lowest quality first
+  # css query [title="I like this"] > span
   likes = doc.xpath_node(%q(//button[@title="I like this"]/span))
   if likes
-    likes = likes.content
+    likes = likes.content.delete(",").to_i
   else
-    likes = "n/a"
+    likes = 1
   end
-  
+
+  # css query [title="I dislike this"] > span
   dislikes = doc.xpath_node(%q(//button[@title="I dislike this"]/span))
   if dislikes
-    dislikes.content
+    dislikes = dislikes.content.delete(",").to_i
   else
-    dislikes = "n/a"
+    dislikes = 1
   end
 
-  File.write("video_info/#{video_id}", video_info.to_json)
+  engagement = ((dislikes.to_f32 + likes.to_f32)*100 / video_info["view_count"].to_i).to_i
+  calculated_rating = likes.to_f32/(likes.to_f32 + dislikes.to_f32)*4 + 1
+
   templated "watch"
 end
 
-# get "/listen/:video_id" do |env|
-#   video_id = env.params.url["video_id"]
-
-#   video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body
-#   video_info = object_to_hash(video_info_encoded)
-#   File.write("video_info/#{video_id}", video_info.to_json)
-#   templated "listen"
-# end
-
 public_folder "assets"
 
 Kemal.run

views/listen.ecr

@@ -1,6 +1,5 @@
-<% title = URI.unescape(video_info["title"].as(String), true) %>
+<h1><%= video_info["title"] %></h1>
-<h1><%= title %></h1>
+<video style="width: 100%" poster="<%= video_info.has_key?("iurlhq720") ? video_info["iurlhq720"] : video_info["iurl"] %>" controls>
-<video style="width: 100%" poster="<%= video_info["iurlhq720"] %>" controls>
 <% video_info["adaptive_fmts"].as(Hash).each do |key, value| %>
     <% url = value["url"] %>
     <% type = value["type"].to_s.split(";")[0] %>

views/watch.ecr

@@ -1,20 +1,23 @@
-<% title = URI.unescape(video_info["title"].as(String), true) %>
+<video style="width: 100%" poster="<%= video_info.has_key?("iurlhq720") ? video_info["iurlhq720"] : video_info["iurlmq"] %>" controls>
-<h1><%= title %></h1>
+<% fmt_stream.each do |fmt| %>
-<video style="width: 100%" poster="<%= video_info["iurl"] %>" controls>
+    <source src="<%= fmt["url"] %>" type="<%= fmt["type"].split(";")[0] %>">
-<% video_info["url_encoded_fmt_stream_map"].as(Hash).each do |key, value| %>
-    <% url = value["url"] %>
-    <% type = value["type"]["0"].to_s.split(";")[0] %>
-    <source src="<%= url %>" type="<%= type %>">
 <% end %>
 </video>
+<h1><%= video_info["title"] %></h1>
 <div class="pure-g">
     <div class="pure-u-1 pure-u-md-1-5">
-        <p>Likes: <%= likes %></p>
+        <p>+  <%= likes %></p>
-        <p>Dislikes: <%= dislikes %></p>
+        <p>-&nbsp;  <%= dislikes %></p>
     </div>
-    <div class="pure-u-1 pure-u-md-3-5"></div>
+    <div class="pure-u-1 pure-u-md-3-5">
-    <div class="pure-u-1 pure-u-md-1-5">
         <p>Views : <%= video_info["view_count"] %></p>
         <p>Rating : <%= video_info["avg_rating"] %></p>
+        <p>Calculated Rating : <%= calculated_rating %></p>
+        <p>Engagement : <%= engagement %>%</p>
+    </div>
+    <div class="pure-u-1 pure-u-md-1-5">
+    <% fmt_stream.each do |fmt| %>
+        <p><%= fmt["quality"] %></p>
+    <% end %>
     </div>
 </div>

youtube api.md

@@ -38,7 +38,7 @@ https://www.youtube.com/embed/:ID
 
 
 https://www.youtube.com/get_video_info?video_id= + video_id +
-            &el=info&ps=default&eurl=&gl=US&hl=en"&el=info&ps=default&eurl=&gl=US&hl=en
+            &el=info&ps=default&eurl=&gl=US&hl=en
 
 ```