From 115a8155285a37fe70d3925a5b3b3a6956265d28 Mon Sep 17 00:00:00 2001 From: Omar Roth Date: Mon, 23 Jul 2018 15:09:11 -0500 Subject: [PATCH] Add `/videos/:id' --- src/invidious.cr | 233 +++++++++++++++++++++++++++++++++++++-- src/invidious/helpers.cr | 100 +++++++++++++++++ 2 files changed, 326 insertions(+), 7 deletions(-) diff --git a/src/invidious.cr b/src/invidious.cr index d7e1da46..073b444e 100644 --- a/src/invidious.cr +++ b/src/invidious.cr @@ -458,8 +458,8 @@ get "/captions/:id" do |env| env.response.content_type = "text/vtt" if track.empty? - halt env, status_code: 403 - else + halt env, status_code: 403 + else track = track[0] end @@ -583,11 +583,15 @@ get "/comments/:id" do |env| content_text ||= item_comment["contentText"]["runs"][0]["text"].as_s.rchop('\ufeff') json.field "author", item_comment["authorText"]["simpleText"] - json.field "authorThumbnail" do - json.object do - json.field "url", item_comment["authorThumbnail"]["thumbnails"][-1]["url"] - json.field "width", item_comment["authorThumbnail"]["thumbnails"][-1]["width"] - json.field "height", item_comment["authorThumbnail"]["thumbnails"][-1]["height"] + json.field "authorThumbnails" do + json.array do + item_comment["authorThumbnail"]["thumbnails"].as_a.each do |thumbnail| + json.object do + json.field "url", thumbnail["url"] + json.field "width", thumbnail["width"] + json.field "height", thumbnail["height"] + end + end end end json.field "authorId", item_comment["authorEndpoint"]["browseEndpoint"]["browseId"] @@ -649,6 +653,221 @@ get "/comments/:id" do |env| end end +get "/videos/:id" do |env| + id = env.params.url["id"] + + client = make_client(YT_URL) + begin + video = get_video(id, client, PG_DB) + rescue ex + halt env, status_code: 403 + end + + adaptive_fmts = [] of HTTP::Params + if video.info.has_key?("adaptive_fmts") + video.info["adaptive_fmts"].split(",") do |string| + adaptive_fmts << HTTP::Params.parse(string) + end + end + + fmt_stream = [] of HTTP::Params + video.info["url_encoded_fmt_stream_map"].split(",") do |string| + if !string.empty? + fmt_stream << HTTP::Params.parse(string) + end + end + + if adaptive_fmts[0]? && adaptive_fmts[0]["s"]? + adaptive_fmts.each do |fmt| + fmt["url"] += "&signature=" + decrypt_signature(fmt["s"], decrypt_function) + end + + fmt_stream.each do |fmt| + fmt["url"] += "&signature=" + decrypt_signature(fmt["s"], decrypt_function) + end + end + + player_response = JSON.parse(video.info["player_response"]) + if player_response["captions"]? + captions = player_response["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]?.try &.as_a + end + captions ||= [] of JSON::Any + + env.response.content_type = "application/json" + video_info = JSON.build do |json| + json.object do + json.field "title", video.title + json.field "videoId", video.id + json.field "videoThumbnails" do + json.object do + qualities = [{name: "default", url: "default", width: 120, height: 90}, + {name: "high", url: "hqdefault", width: 480, height: 360}, + {name: "medium", url: "mqdefault", width: 320, height: 180}, + ] + qualities.each do |quality| + json.field quality[:name] do + json.object do + json.field "url", "https://i.ytimg.com/vi/#{id}/#{quality["url"]}.jpg" + json.field "width", quality[:width] + json.field "height", quality[:height] + end + end + end + end + end + + description = video.description.gsub("
", "\n") + description = description.gsub("
", "\n") + description = XML.parse_html(description) + + json.field "description", description.content + json.field "descriptionHtml", video.description + json.field "published", video.published.epoch + json.field "keywords" do + json.array do + video.info["keywords"].split(",").each { |keyword| json.string keyword } + end + end + + json.field "viewCount", video.views + json.field "likeCount", video.likes + json.field "dislikeCount", video.dislikes + + json.field "author", video.author + json.field "authorId", video.ucid + json.field "authorUrl", "/channel/#{video.ucid}" + + json.field "lengthSeconds", video.info["length_seconds"].to_i + if video.info["allow_ratings"]? + json.field "allowRatings", video.info["allow_ratings"] == "1" + else + json.field "allowRatings", false + end + json.field "rating", video.info["avg_rating"].to_f32 + + if video.info["is_listed"]? + json.field "isListed", video.info["is_listed"] == "1" + end + + fmt_list = video.info["fmt_list"].split(",").map { |fmt| fmt.split("/")[1] } + fmt_list = Hash.zip(fmt_list.map { |fmt| fmt[0] }, fmt_list.map { |fmt| fmt[1] }) + + json.field "adaptiveFormats" do + json.array do + adaptive_fmts.each_with_index do |adaptive_fmt, i| + json.object do + json.field "index", adaptive_fmt["index"] + json.field "bitrate", adaptive_fmt["bitrate"] + json.field "init", adaptive_fmt["init"] + json.field "url", adaptive_fmt["url"] + json.field "itag", adaptive_fmt["itag"] + json.field "type", adaptive_fmt["type"] + json.field "clen", adaptive_fmt["clen"] + json.field "lmt", adaptive_fmt["lmt"] + json.field "projectionType", adaptive_fmt["projection_type"] + + fmt_info = itag_to_metadata(adaptive_fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + + if fmt_info["fps"]? + json.field "fps", fmt_info["fps"] + end + + if fmt_info["height"]? + json.field "qualityLabel", "#{fmt_info["height"]}p" + json.field "resolution", "#{fmt_info["height"]}p" + + if fmt_info["width"]? + json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" + end + end + end + end + end + end + + json.field "formatStreams" do + json.array do + fmt_stream.each do |fmt| + json.object do + json.field "url", fmt["url"] + json.field "itag", fmt["itag"] + json.field "type", fmt["type"] + json.field "quality", fmt["quality"] + + fmt_info = itag_to_metadata(fmt["itag"]) + json.field "container", fmt_info["ext"] + json.field "encoding", fmt_info["vcodec"]? || fmt_info["acodec"] + + if fmt_info["fps"]? + json.field "fps", fmt_info["fps"] + end + + if fmt_info["height"]? + json.field "qualityLabel", "#{fmt_info["height"]}p" + json.field "resolution", "#{fmt_info["height"]}p" + + if fmt_info["width"]? + json.field "size", "#{fmt_info["width"]}x#{fmt_info["height"]}" + end + end + end + end + end + end + + json.field "captions" do + json.array do + captions.each do |caption| + json.object do + json.field "label", caption["name"]["simpleText"] + json.field "languageCode", caption["languageCode"] + end + end + end + end + + json.field "recommendedVideos" do + json.array do + video.info["rvs"].split(",").each do |rv| + rv = HTTP::Params.parse(rv) + + if rv["id"]? + json.object do + json.field "videoId", rv["id"] + json.field "title", rv["title"] + json.field "videoThumbnails" do + json.object do + qualities = [{name: "default", url: "default", width: 120, height: 90}, + {name: "high", url: "hqdefault", width: 480, height: 360}, + {name: "medium", url: "mqdefault", width: 320, height: 180}, + ] + qualities.each do |quality| + json.field quality[:name] do + json.object do + json.field "url", "https://i.ytimg.com/vi/#{id}/#{quality["url"]}.jpg" + json.field "width", quality[:width] + json.field "height", quality[:height] + end + end + end + end + end + json.field "author", rv["author"] + json.field "lengthSeconds", rv["length_seconds"] + json.field "viewCountText", rv["short_view_count_text"].rchop(" views") + end + end + end + end + end + end + end + + video_info +end + get "/embed/:id" do |env| if env.params.url["id"]? id = env.params.url["id"] diff --git a/src/invidious/helpers.cr b/src/invidious/helpers.cr index 4f87fc67..184b4f08 100644 --- a/src/invidious/helpers.cr +++ b/src/invidious/helpers.cr @@ -1007,3 +1007,103 @@ def generate_captcha(key) return {challenge: challenge, token: token} end + +def itag_to_metadata(itag : String) + # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 + formats = {"5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, + "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, + "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, + "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, + "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, + "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + + # 3D videos + "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, + "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, + "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, + + # Apple HTTP Live Streaming + "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, + "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, + "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, + "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, + + # DASH mp4 video + "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, + "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, + "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, + "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, + "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https=>//github.com/rg3/youtube-dl/issues/4559) + "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, + "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, + "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, + "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, + "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, + + # Dash mp4 audio + "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, + "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, + "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, + "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, + "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, + "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, + + # Dash webm + "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, + "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, + "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, + "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, + "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, + "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, + "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, + "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, + # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) + "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, + "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, + + # Dash webm audio + "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, + "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, + + # Dash webm audio with opus inside + "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, + "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, + "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, + } + + return formats[itag] +end