From 7feec0c00dff72378fd80723c16d52803c6d882c Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Tue, 28 Nov 2017 19:59:51 -0600 Subject: [PATCH] Add engagement rating, rating, likes, dislikes, and form of logging (primitive but in the hope of reverse-engineering requests) --- shard.lock | 2 +- shard.yml | 1 + src/visor.cr | 90 ++++++++++++++++-------------------------------- views/listen.ecr | 5 ++- views/watch.ecr | 25 ++++++++------ youtube api.md | 2 +- 6 files changed, 49 insertions(+), 76 deletions(-) diff --git a/shard.lock b/shard.lock index 1e791997..2c867ae4 100644 --- a/shard.lock +++ b/shard.lock @@ -6,7 +6,7 @@ shards: kemal: github: kemalcr/kemal - version: 0.21.0 + commit: 8cb9770ec3c6cf5897e644229dad8d0b5c360941 kilt: github: jeromegn/kilt diff --git a/shard.yml b/shard.yml index dc5cb756..c0be6d9b 100644 --- a/shard.yml +++ b/shard.yml @@ -11,6 +11,7 @@ targets: dependencies: kemal: github: kemalcr/kemal + branch: master pg: github: will/crystal-pg diff --git a/src/visor.cr b/src/visor.cr index 77e7e170..1b9a9b2d 100644 --- a/src/visor.cr +++ b/src/visor.cr @@ -3,93 +3,63 @@ require "json" require "kemal" require "pg" require "xml" +require "./url_encoded" macro templated(filename) render "views/#{{{filename}}}.ecr", "views/layout.ecr" end -# pg = DB.open("postgres://kemal@visor/dev") - -alias Type = String | Hash(String, Type) - -def object_to_hash(value) - object = {} of String => Type - items = value.split("&") - items.each do |item| - key, value = item.split("=") - value = URI.unescape(value) - object[key] = parse_uri(value) - end - return object -end - -def array_to_hash(value) - array = {} of String => Type - items = value.split(",") - count = 0 - items.each do |item| - array[count.to_s] = parse_uri(item) - count += 1 - end - return array -end - -def parse_uri(value) - if value.starts_with?("http") || value.starts_with?("[") - return value - else - if value.includes?(",") - return array_to_hash(value) - elsif value.includes?("&") - return object_to_hash(value) - else - return value - end - end -end - context = OpenSSL::SSL::Context::Client.insecure -client = HTTP::Client.new("www.youtube.com", 443, context) +fmt_file = File.open("temp/fmt_stream") get "/" do |env| templated "index" end + get "/watch/:video_id" do |env| video_id = env.params.url["video_id"] - video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body - video_info = object_to_hash(video_info_encoded) - body = client.get("/watch?v=#{video_id}").body - doc = XML.parse(body) + client = HTTP::Client.new("www.youtube.com", 443, context) + video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body + video_info = HTTP::Params.parse(video_info) + pageContent = client.get("/watch?v=#{video_id}").body + doc = XML.parse(pageContent) + fmt_stream = [] of HTTP::Params + video_info["url_encoded_fmt_stream_map"].split(",") do |string| + fmt_stream << HTTP::Params.parse(string) + end + + File.write("temp/#{video_id}", video_info) + File.write("temp/#{video_id}_manifest", video_info["dashmpd"]) + File.open("temp/#{video_id}_fmt_stream_0", "a+").puts fmt_stream[0]["url"] + File.open("temp/#{video_id}_fmt_stream_1", "a+").puts fmt_stream[1]["url"] + File.open("temp/#{video_id}_fmt_stream_2", "a+").puts fmt_stream[2]["url"] + File.open("temp/#{video_id}_fmt_stream_3", "a+").puts fmt_stream[3]["url"] + fmt_stream.reverse! # We want lowest quality first + # css query [title="I like this"] > span likes = doc.xpath_node(%q(//button[@title="I like this"]/span)) if likes - likes = likes.content + likes = likes.content.delete(",").to_i else - likes = "n/a" + likes = 1 end - + + # css query [title="I dislike this"] > span dislikes = doc.xpath_node(%q(//button[@title="I dislike this"]/span)) if dislikes - dislikes.content + dislikes = dislikes.content.delete(",").to_i else - dislikes = "n/a" + dislikes = 1 end - File.write("video_info/#{video_id}", video_info.to_json) + engagement = ((dislikes.to_f32 + likes.to_f32)*100 / video_info["view_count"].to_i).to_i + calculated_rating = likes.to_f32/(likes.to_f32 + dislikes.to_f32)*4 + 1 + templated "watch" end -# get "/listen/:video_id" do |env| -# video_id = env.params.url["video_id"] - -# video_info_encoded = HTTP::Client.get("https://www.youtube.com/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en", nil, nil, tls = context).body -# video_info = object_to_hash(video_info_encoded) -# File.write("video_info/#{video_id}", video_info.to_json) -# templated "listen" -# end - public_folder "assets" Kemal.run diff --git a/views/listen.ecr b/views/listen.ecr index c8040f14..97502fb6 100644 --- a/views/listen.ecr +++ b/views/listen.ecr @@ -1,6 +1,5 @@ -<% title = URI.unescape(video_info["title"].as(String), true) %> -

<%= title %>

-