CAPTION_LANGUAGES = { "", "English", "English (auto-generated)", "Afrikaans", "Albanian", "Amharic", "Arabic", "Armenian", "Azerbaijani", "Bangla", "Basque", "Belarusian", "Bosnian", "Bulgarian", "Burmese", "Catalan", "Cebuano", "Chinese (Simplified)", "Chinese (Traditional)", "Corsican", "Croatian", "Czech", "Danish", "Dutch", "Esperanto", "Estonian", "Filipino", "Finnish", "French", "Galician", "Georgian", "German", "Greek", "Gujarati", "Haitian Creole", "Hausa", "Hawaiian", "Hebrew", "Hindi", "Hmong", "Hungarian", "Icelandic", "Igbo", "Indonesian", "Irish", "Italian", "Japanese", "Javanese", "Kannada", "Kazakh", "Khmer", "Korean", "Kurdish", "Kyrgyz", "Lao", "Latin", "Latvian", "Lithuanian", "Luxembourgish", "Macedonian", "Malagasy", "Malay", "Malayalam", "Maltese", "Maori", "Marathi", "Mongolian", "Nepali", "Norwegian", "Nyanja", "Pashto", "Persian", "Polish", "Portuguese", "Punjabi", "Romanian", "Russian", "Samoan", "Scottish Gaelic", "Serbian", "Shona", "Sindhi", "Sinhala", "Slovak", "Slovenian", "Somali", "Southern Sotho", "Spanish", "Spanish (Latin America)", "Sundanese", "Swahili", "Swedish", "Tajik", "Tamil", "Telugu", "Thai", "Turkish", "Ukrainian", "Urdu", "Uzbek", "Vietnamese", "Welsh", "Western Frisian", "Xhosa", "Yiddish", "Yoruba", "Zulu", } REGIONS = {"AD", "AE", "AF", "AG", "AI", "AL", "AM", "AO", "AQ", "AR", "AS", "AT", "AU", "AW", "AX", "AZ", "BA", "BB", "BD", "BE", "BF", "BG", "BH", "BI", "BJ", "BL", "BM", "BN", "BO", "BQ", "BR", "BS", "BT", "BV", "BW", "BY", "BZ", "CA", "CC", "CD", "CF", "CG", "CH", "CI", "CK", "CL", "CM", "CN", "CO", "CR", "CU", "CV", "CW", "CX", "CY", "CZ", "DE", "DJ", "DK", "DM", "DO", "DZ", "EC", "EE", "EG", "EH", "ER", "ES", "ET", "FI", "FJ", "FK", "FM", "FO", "FR", "GA", "GB", "GD", "GE", "GF", "GG", "GH", "GI", "GL", "GM", "GN", "GP", "GQ", "GR", "GS", "GT", "GU", "GW", "GY", "HK", "HM", "HN", "HR", "HT", "HU", "ID", "IE", "IL", "IM", "IN", "IO", "IQ", "IR", "IS", "IT", "JE", "JM", "JO", "JP", "KE", "KG", "KH", "KI", "KM", "KN", "KP", "KR", "KW", "KY", "KZ", "LA", "LB", "LC", "LI", "LK", "LR", "LS", "LT", "LU", "LV", "LY", "MA", "MC", "MD", "ME", "MF", "MG", "MH", "MK", "ML", "MM", "MN", "MO", "MP", "MQ", "MR", "MS", "MT", "MU", "MV", "MW", "MX", "MY", "MZ", "NA", "NC", "NE", "NF", "NG", "NI", "NL", "NO", "NP", "NR", "NU", "NZ", "OM", "PA", "PE", "PF", "PG", "PH", "PK", "PL", "PM", "PN", "PR", "PS", "PT", "PW", "PY", "QA", "RE", "RO", "RS", "RU", "RW", "SA", "SB", "SC", "SD", "SE", "SG", "SH", "SI", "SJ", "SK", "SL", "SM", "SN", "SO", "SR", "SS", "ST", "SV", "SX", "SY", "SZ", "TC", "TD", "TF", "TG", "TH", "TJ", "TK", "TL", "TM", "TN", "TO", "TR", "TT", "TV", "TW", "TZ", "UA", "UG", "UM", "US", "UY", "UZ", "VA", "VC", "VE", "VG", "VI", "VN", "VU", "WF", "WS", "YE", "YT", "ZA", "ZM", "ZW"} BYPASS_REGIONS = { "GB", "DE", "FR", "IN", "CN", "RU", "CA", "JP", "IT", "TH", "ES", "AE", "KR", "IR", "BR", "PK", "ID", "BD", "MX", "PH", "EG", "VN", "CD", "TR", } VIDEO_THUMBNAILS = { {name: "maxres", host: "invidio.us", url: "maxres", height: 720, width: 1280}, {name: "maxresdefault", host: "i.ytimg.com", url: "maxresdefault", height: 720, width: 1280}, {name: "sddefault", host: "i.ytimg.com", url: "sddefault", height: 480, width: 640}, {name: "high", host: "i.ytimg.com", url: "hqdefault", height: 360, width: 480}, {name: "medium", host: "i.ytimg.com", url: "mqdefault", height: 180, width: 320}, {name: "default", host: "i.ytimg.com", url: "default", height: 90, width: 120}, {name: "start", host: "i.ytimg.com", url: "1", height: 90, width: 120}, {name: "middle", host: "i.ytimg.com", url: "2", height: 90, width: 120}, {name: "end", host: "i.ytimg.com", url: "3", height: 90, width: 120}, } # See https://github.com/rg3/youtube-dl/blob/master/youtube_dl/extractor/youtube.py#L380-#L476 VIDEO_FORMATS = { "5" => {"ext" => "flv", "width" => 400, "height" => 240, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, "6" => {"ext" => "flv", "width" => 450, "height" => 270, "acodec" => "mp3", "abr" => 64, "vcodec" => "h263"}, "13" => {"ext" => "3gp", "acodec" => "aac", "vcodec" => "mp4v"}, "17" => {"ext" => "3gp", "width" => 176, "height" => 144, "acodec" => "aac", "abr" => 24, "vcodec" => "mp4v"}, "18" => {"ext" => "mp4", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 96, "vcodec" => "h264"}, "22" => {"ext" => "mp4", "width" => 1280, "height" => 720, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "34" => {"ext" => "flv", "width" => 640, "height" => 360, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "35" => {"ext" => "flv", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "36" => {"ext" => "3gp", "width" => 320, "acodec" => "aac", "vcodec" => "mp4v"}, "37" => {"ext" => "mp4", "width" => 1920, "height" => 1080, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "38" => {"ext" => "mp4", "width" => 4096, "height" => 3072, "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "43" => {"ext" => "webm", "width" => 640, "height" => 360, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "44" => {"ext" => "webm", "width" => 854, "height" => 480, "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "45" => {"ext" => "webm", "width" => 1280, "height" => 720, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "46" => {"ext" => "webm", "width" => 1920, "height" => 1080, "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "59" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "78" => {"ext" => "mp4", "width" => 854, "height" => 480, "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, # 3D videos "82" => {"ext" => "mp4", "height" => 360, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "83" => {"ext" => "mp4", "height" => 480, "format" => "3D", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "84" => {"ext" => "mp4", "height" => 720, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "85" => {"ext" => "mp4", "height" => 1080, "format" => "3D", "acodec" => "aac", "abr" => 192, "vcodec" => "h264"}, "100" => {"ext" => "webm", "height" => 360, "format" => "3D", "acodec" => "vorbis", "abr" => 128, "vcodec" => "vp8"}, "101" => {"ext" => "webm", "height" => 480, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, "102" => {"ext" => "webm", "height" => 720, "format" => "3D", "acodec" => "vorbis", "abr" => 192, "vcodec" => "vp8"}, # Apple HTTP Live Streaming "91" => {"ext" => "mp4", "height" => 144, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "92" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "93" => {"ext" => "mp4", "height" => 360, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "94" => {"ext" => "mp4", "height" => 480, "format" => "HLS", "acodec" => "aac", "abr" => 128, "vcodec" => "h264"}, "95" => {"ext" => "mp4", "height" => 720, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, "96" => {"ext" => "mp4", "height" => 1080, "format" => "HLS", "acodec" => "aac", "abr" => 256, "vcodec" => "h264"}, "132" => {"ext" => "mp4", "height" => 240, "format" => "HLS", "acodec" => "aac", "abr" => 48, "vcodec" => "h264"}, "151" => {"ext" => "mp4", "height" => 72, "format" => "HLS", "acodec" => "aac", "abr" => 24, "vcodec" => "h264"}, # DASH mp4 video "133" => {"ext" => "mp4", "height" => 240, "format" => "DASH video", "vcodec" => "h264"}, "134" => {"ext" => "mp4", "height" => 360, "format" => "DASH video", "vcodec" => "h264"}, "135" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, "136" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264"}, "137" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264"}, "138" => {"ext" => "mp4", "format" => "DASH video", "vcodec" => "h264"}, # Height can vary (https=>//github.com/rg3/youtube-dl/issues/4559) "160" => {"ext" => "mp4", "height" => 144, "format" => "DASH video", "vcodec" => "h264"}, "212" => {"ext" => "mp4", "height" => 480, "format" => "DASH video", "vcodec" => "h264"}, "264" => {"ext" => "mp4", "height" => 1440, "format" => "DASH video", "vcodec" => "h264"}, "298" => {"ext" => "mp4", "height" => 720, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, "299" => {"ext" => "mp4", "height" => 1080, "format" => "DASH video", "vcodec" => "h264", "fps" => 60}, "266" => {"ext" => "mp4", "height" => 2160, "format" => "DASH video", "vcodec" => "h264"}, # Dash mp4 audio "139" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 48, "container" => "m4a_dash"}, "140" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 128, "container" => "m4a_dash"}, "141" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "abr" => 256, "container" => "m4a_dash"}, "256" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, "258" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "aac", "container" => "m4a_dash"}, "325" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "dtse", "container" => "m4a_dash"}, "328" => {"ext" => "m4a", "format" => "DASH audio", "acodec" => "ec-3", "container" => "m4a_dash"}, # Dash webm "167" => {"ext" => "webm", "height" => 360, "width" => 640, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "168" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "169" => {"ext" => "webm", "height" => 720, "width" => 1280, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "170" => {"ext" => "webm", "height" => 1080, "width" => 1920, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "218" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "219" => {"ext" => "webm", "height" => 480, "width" => 854, "format" => "DASH video", "container" => "webm", "vcodec" => "vp8"}, "278" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "container" => "webm", "vcodec" => "vp9"}, "242" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9"}, "243" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9"}, "244" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "245" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "246" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9"}, "247" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9"}, "248" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9"}, "271" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9"}, # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug) "272" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, "302" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "303" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "308" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "313" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9"}, "315" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "330" => {"ext" => "webm", "height" => 144, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "331" => {"ext" => "webm", "height" => 240, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "332" => {"ext" => "webm", "height" => 360, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "333" => {"ext" => "webm", "height" => 480, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "334" => {"ext" => "webm", "height" => 720, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "335" => {"ext" => "webm", "height" => 1080, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "336" => {"ext" => "webm", "height" => 1440, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, "337" => {"ext" => "webm", "height" => 2160, "format" => "DASH video", "vcodec" => "vp9", "fps" => 60}, # Dash webm audio "171" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 128}, "172" => {"ext" => "webm", "acodec" => "vorbis", "format" => "DASH audio", "abr" => 256}, # Dash webm audio with opus inside "249" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 50}, "250" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 70}, "251" => {"ext" => "webm", "format" => "DASH audio", "acodec" => "opus", "abr" => 160}, } class Video property player_json : JSON::Any? module HTTPParamConverter def self.from_rs(rs) HTTP::Params.parse(rs.read(String)) end end def fmt_stream(decrypt_function) streams = [] of HTTP::Params self.info["url_encoded_fmt_stream_map"].split(",") do |string| if !string.empty? streams << HTTP::Params.parse(string) end end streams.each { |s| s.add("label", "#{s["quality"]} - #{s["type"].split(";")[0].split("/")[1]}") } streams = streams.uniq { |s| s["label"] } if self.info["region"]? streams.each do |fmt| fmt["url"] += "®ion=" + self.info["region"] end end if streams[0]? && streams[0]["s"]? streams.each do |fmt| fmt["url"] += "&signature=" + decrypt_signature(fmt["s"], decrypt_function) end end return streams end def adaptive_fmts(decrypt_function) adaptive_fmts = [] of HTTP::Params if self.info.has_key?("adaptive_fmts") self.info["adaptive_fmts"].split(",") do |string| adaptive_fmts << HTTP::Params.parse(string) end elsif self.info.has_key?("dashmpd") client = make_client(YT_URL) response = client.get(self.info["dashmpd"]) document = XML.parse_html(response.body) document.xpath_nodes(%q(//adaptationset)).each do |adaptation_set| mime_type = adaptation_set["mimetype"] document.xpath_nodes(%q(.//representation)).each do |representation| codecs = representation["codecs"] itag = representation["id"] bandwidth = representation["bandwidth"] url = representation.xpath_node(%q(.//baseurl)).not_nil!.content clen = url.match(/clen\/(?\d+)/).try &.["clen"] clen ||= "0" lmt = url.match(/lmt\/(?\d+)/).try &.["lmt"] lmt ||= "#{((Time.now + 1.hour).epoch_f.to_f64 * 1000000).to_i64}" segment_list = representation.xpath_node(%q(.//segmentlist)).not_nil! init = segment_list.xpath_node(%q(.//initialization)) # TODO: Replace with sane defaults when byteranges are absent if init && !init["sourceurl"].starts_with? "sq" init = init["sourceurl"].lchop("range/") index = segment_list.xpath_node(%q(.//segmenturl)).not_nil!["media"] index = index.lchop("range/") index = "#{init.split("-")[1].to_i + 1}-#{index.split("-")[0].to_i}" else init = "0-0" index = "1-1" end params = { "type" => ["#{mime_type}; codecs=\"#{codecs}\""], "url" => [url], "projection_type" => ["1"], "index" => [index], "init" => [init], "xtags" => [] of String, "lmt" => [lmt], "clen" => [clen], "bitrate" => [bandwidth], "itag" => [itag], } if mime_type == "video/mp4" width = representation["width"]? height = representation["height"]? fps = representation["framerate"]? metadata = itag_to_metadata?(itag) if metadata width ||= metadata["width"]? height ||= metadata["height"]? fps ||= metadata["fps"]? end if width && height params["size"] = ["#{width}x#{height}"] end if width params["quality_label"] = ["#{height}p"] end end adaptive_fmts << HTTP::Params.new(params) end end end if self.info["region"]? adaptive_fmts.each do |fmt| fmt["url"] += "®ion=" + self.info["region"] end end if adaptive_fmts[0]? && adaptive_fmts[0]["s"]? adaptive_fmts.each do |fmt| fmt["url"] += "&signature=" + decrypt_signature(fmt["s"], decrypt_function) end end return adaptive_fmts end def video_streams(adaptive_fmts) video_streams = adaptive_fmts.compact_map { |s| s["type"].starts_with?("video") ? s : nil } return video_streams end def audio_streams(adaptive_fmts) audio_streams = adaptive_fmts.compact_map { |s| s["type"].starts_with?("audio") ? s : nil } audio_streams.sort_by! { |s| s["bitrate"].to_i }.reverse! audio_streams.each do |stream| stream["bitrate"] = (stream["bitrate"].to_f64/1000).to_i.to_s end return audio_streams end def player_response if !@player_json @player_json = JSON.parse(@info["player_response"]) end return @player_json.not_nil! end def paid reason = self.player_response["playabilityStatus"]?.try &.["reason"]? if reason == "This video requires payment to watch." paid = true else paid = false end return paid end def premium premium = self.player_response.to_s.includes? "Get YouTube without the ads." return premium end def captions captions = [] of Caption if player_response["captions"]? caption_list = player_response["captions"]["playerCaptionsTracklistRenderer"]["captionTracks"]?.try &.as_a caption_list ||= [] of JSON::Any caption_list.each do |caption| caption = Caption.from_json(caption.to_json) caption.name.simpleText = caption.name.simpleText.split(" - ")[0] captions << caption end end return captions end def short_description description = self.description.gsub("
", " ") description = description.gsub("
", " ") description = XML.parse_html(description).content[0..200].gsub('"', """).gsub("\n", " ").strip(" ") if description.empty? description = " " end return description end add_mapping({ id: String, info: { type: HTTP::Params, default: HTTP::Params.parse(""), converter: Video::HTTPParamConverter, }, updated: Time, title: String, views: Int64, likes: Int32, dislikes: Int32, wilson_score: Float64, published: Time, description: String, language: String?, author: String, ucid: String, allowed_regions: Array(String), is_family_friendly: Bool, genre: String, genre_url: String, license: String, sub_count_text: String, author_thumbnail: { type: String, default: "", }, }) end class Caption JSON.mapping( name: CaptionName, baseUrl: String, languageCode: String ) end class CaptionName JSON.mapping( simpleText: String, ) end class VideoRedirect < Exception end def get_video(id, db, proxies = {} of String => Array({ip: String, port: Int32}), refresh = true) if db.query_one?("SELECT EXISTS (SELECT true FROM videos WHERE id = $1)", id, as: Bool) video = db.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video) # If record was last updated over 10 minutes ago, refresh (expire param in response lasts for 6 hours) if refresh && Time.now - video.updated > 10.minutes begin video = fetch_video(id, proxies) video_array = video.to_a args = arg_array(video_array[1..-1], 2) db.exec("UPDATE videos SET (info,updated,title,views,likes,dislikes,wilson_score,\ published,description,language,author,ucid,allowed_regions,is_family_friendly,\ genre,genre_url,license,sub_count_text,author_thumbnail)\ = (#{args}) WHERE id = $1", video_array) rescue ex db.exec("DELETE FROM videos * WHERE id = $1", id) raise ex end end else video = fetch_video(id, proxies) video_array = video.to_a args = arg_array(video_array) db.exec("INSERT INTO videos VALUES (#{args}) ON CONFLICT (id) DO NOTHING", video_array) end return video end def fetch_video(id, proxies) html_channel = Channel(XML::Node | String).new info_channel = Channel(HTTP::Params).new spawn do client = make_client(YT_URL) html = client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1") if md = html.headers["location"]?.try &.match(/v=(?[a-zA-Z0-9_-]{11})/) next html_channel.send(md["id"]) end html = XML.parse_html(html.body) html_channel.send(html) end spawn do client = make_client(YT_URL) info = client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1") info = HTTP::Params.parse(info.body) if info["reason"]? info = client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1") info = HTTP::Params.parse(info.body) end info_channel.send(info) end html = html_channel.receive if html.as?(String) raise VideoRedirect.new("#{html.as(String)}") end html = html.as(XML::Node) info = info_channel.receive if info["reason"]? && info["reason"].includes? "your country" bypass_channel = Channel(HTTPProxy | Nil).new proxies.each do |region, list| spawn do info = HTTP::Params.new({ "reason" => [info["reason"]], }) list.each do |proxy| begin client = HTTPClient.new(YT_URL) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds proxy = HTTPProxy.new(proxy_host: proxy[:ip], proxy_port: proxy[:port]) client.set_proxy(proxy) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) if !info["reason"]? bypass_channel.send(proxy) break end rescue ex end end # If none of the proxies we tried returned a valid response if info["reason"]? bypass_channel.send(nil) end end end proxies.size.times do proxy = bypass_channel.receive if proxy begin client = HTTPClient.new(YT_URL) client.read_timeout = 10.seconds client.connect_timeout = 10.seconds client.set_proxy(proxy) html = XML.parse_html(client.get("/watch?v=#{id}&bpctr=#{Time.new.epoch + 2000}&gl=US&hl=en&disable_polymer=1").body) info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&el=detailpage&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) if info["reason"]? info = HTTP::Params.parse(client.get("/get_video_info?video_id=#{id}&ps=default&eurl=&gl=US&hl=en&disable_polymer=1").body) end proxy = {ip: proxy.proxy_host, port: proxy.proxy_port} region = proxies.select { |region, list| list.includes? proxy } if !region.empty? info["region"] = region.keys[0] end break rescue ex end end end end if info["reason"]? raise info["reason"] end title = info["title"] views = info["view_count"].to_i64 author = info["author"] ucid = info["ucid"] likes = html.xpath_node(%q(//button[@title="I like this"]/span)) likes = likes.try &.content.delete(",").try &.to_i likes ||= 0 dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span)) dislikes = dislikes.try &.content.delete(",").try &.to_i dislikes ||= 0 description = html.xpath_node(%q(//p[@id="eow-description"])) description = description ? description.to_xml : "" wilson_score = ci_lower_bound(likes, likes + dislikes) published = html.xpath_node(%q(//meta[@itemprop="datePublished"])).not_nil!["content"] published = Time.parse(published, "%Y-%m-%d", Time::Location.local) allowed_regions = html.xpath_node(%q(//meta[@itemprop="regionsAllowed"])).try &.["content"].split(",") allowed_regions ||= [] of String is_family_friendly = html.xpath_node(%q(//meta[@itemprop="isFamilyFriendly"])).try &.["content"] == "True" is_family_friendly ||= true genre = html.xpath_node(%q(//meta[@itemprop="genre"])).not_nil!["content"] genre_url = html.xpath_node(%(//a[text()="#{genre}"])).try &.["href"] case genre when "Movies" genre_url = "/channel/UClgRkhTL3_hImCAmdLfDE4g" when "Education" # Education channel is linked but does not exist # genre_url = "/channel/UC3yA8nDwraeOfnYfBWun83g" genre_url = "" end genre_url ||= "" license = html.xpath_node(%q(//h4[contains(text(),"License")]/parent::*/ul/li)) if license license = license.content else license = "" end sub_count_text = html.xpath_node(%q(//span[contains(@class, "yt-subscriber-count")])) if sub_count_text sub_count_text = sub_count_text["title"] else sub_count_text = "0" end author_thumbnail = html.xpath_node(%(//img[@alt="#{author}"])) if author_thumbnail author_thumbnail = author_thumbnail["data-thumb"] else author_thumbnail = "" end video = Video.new(id, info, Time.now, title, views, likes, dislikes, wilson_score, published, description, nil, author, ucid, allowed_regions, is_family_friendly, genre, genre_url, license, sub_count_text, author_thumbnail) return video end def itag_to_metadata?(itag : String) return VIDEO_FORMATS[itag]? end def process_video_params(query, preferences) autoplay = query["autoplay"]?.try &.to_i? preferred_captions = query["subtitles"]?.try &.split(",").map { |a| a.downcase } quality = query["quality"]? speed = query["speed"]?.try &.to_f? video_loop = query["loop"]?.try &.to_i? volume = query["volume"]?.try &.to_i? if preferences autoplay ||= preferences.autoplay.to_unsafe preferred_captions ||= preferences.captions quality ||= preferences.quality speed ||= preferences.speed video_loop ||= preferences.video_loop.to_unsafe volume ||= preferences.volume end autoplay ||= 0 preferred_captions ||= [] of String quality ||= "hd720" speed ||= 1 video_loop ||= 0 volume ||= 100 autoplay = autoplay == 1 video_loop = video_loop == 1 if query["t"]? video_start = decode_time(query["t"]) end video_start ||= 0 if query["time_continue"]? video_start = decode_time(query["time_continue"]) end video_start ||= 0 if query["start"]? video_start = decode_time(query["start"]) end if query["end"]? video_end = decode_time(query["end"]) end video_end ||= -1 if query["listen"]? && (query["listen"] == "true" || query["listen"] == "1") listen = true end listen ||= false raw = query["raw"]?.try &.to_i? raw ||= 0 raw = raw == 1 controls = query["controls"]?.try &.to_i? controls ||= 1 controls = controls == 1 params = { autoplay: autoplay, controls: controls, listen: listen, preferred_captions: preferred_captions, quality: quality, raw: raw, speed: speed, video_end: video_end, video_loop: video_loop, video_start: video_start, volume: volume, } return params end def generate_thumbnails(json, id) json.array do VIDEO_THUMBNAILS.each do |thumbnail| json.object do json.field "quality", thumbnail[:name] json.field "url", "https://#{thumbnail[:host]}/vi/#{id}/#{thumbnail["url"]}.jpg" json.field "width", thumbnail[:width] json.field "height", thumbnail[:height] end end end end