Split helpers.cr into multiple files

src/invidious/helpers/helpers.cr

@@ -0,0 +1,273 @@
+class Config
+  YAML.mapping({
+    crawl_threads:   Int32,
+    channel_threads: Int32,
+    video_threads:   Int32,
+    db:              NamedTuple(
+      user: String,
+      password: String,
+      host: String,
+      port: Int32,
+      dbname: String,
+    ),
+    dl_api_key: String?,
+    https_only: Bool?,
+    hmac_key:   String?,
+  })
+end
+
+class FilteredCompressHandler < Kemal::Handler
+  exclude ["/videoplayback", "/api/*"]
+
+  def call(env)
+    return call_next env if exclude_match? env
+
+    {% if flag?(:without_zlib) %}
+      call_next env
+    {% else %}
+      request_headers = env.request.headers
+
+      if request_headers.includes_word?("Accept-Encoding", "gzip")
+        env.response.headers["Content-Encoding"] = "gzip"
+        env.response.output = Gzip::Writer.new(env.response.output, sync_close: true)
+      elsif request_headers.includes_word?("Accept-Encoding", "deflate")
+        env.response.headers["Content-Encoding"] = "deflate"
+        env.response.output = Flate::Writer.new(env.response.output, sync_close: true)
+      end
+
+      call_next env
+    {% end %}
+  end
+end
+
+def rank_videos(db, n, filter, url)
+  top = [] of {Float64, String}
+
+  db.query("SELECT id, wilson_score, published FROM videos WHERE views > 5000 ORDER BY published DESC LIMIT 1000") do |rs|
+    rs.each do
+      id = rs.read(String)
+      wilson_score = rs.read(Float64)
+      published = rs.read(Time)
+
+      # Exponential decay, older videos tend to rank lower
+      temperature = wilson_score * Math.exp(-0.000005*((Time.now - published).total_minutes))
+      top << {temperature, id}
+    end
+  end
+
+  top.sort!
+
+  # Make hottest come first
+  top.reverse!
+  top = top.map { |a, b| b }
+
+  if filter
+    language_list = [] of String
+    top.each do |id|
+      if language_list.size == n
+        break
+      else
+        client = make_client(url)
+        begin
+          video = get_video(id, db)
+        rescue ex
+          next
+        end
+
+        if video.language
+          language = video.language
+        else
+          description = XML.parse(video.description)
+          content = [video.title, description.content].join(" ")
+          content = content[0, 10000]
+
+          results = DetectLanguage.detect(content)
+          language = results[0].language
+
+          db.exec("UPDATE videos SET language = $1 WHERE id = $2", language, id)
+        end
+
+        if language == "en"
+          language_list << id
+        end
+      end
+    end
+    return language_list
+  else
+    return top[0..n - 1]
+  end
+end
+
+def login_req(login_form, f_req)
+  data = {
+    "pstMsg"          => "1",
+    "checkConnection" => "youtube",
+    "checkedDomains"  => "youtube",
+    "hl"              => "en",
+    "deviceinfo"      => %q([null,null,null,[],null,"US",null,null,[],"GlifWebSignIn",null,[null,null,[]]]),
+    "f.req"           => f_req,
+    "flowName"        => "GlifWebSignIn",
+    "flowEntry"       => "ServiceLogin",
+  }
+
+  data = login_form.merge(data)
+
+  return HTTP::Params.encode(data)
+end
+
+def produce_playlist_url(ucid, index)
+  ucid = ucid.lchop("UC")
+  ucid = "VLUU" + ucid
+
+  continuation = write_var_int(index)
+  continuation.unshift(0x08_u8)
+  slice = continuation.to_unsafe.to_slice(continuation.size)
+
+  continuation = Base64.urlsafe_encode(slice, false)
+  continuation = "PT:" + continuation
+  continuation = continuation.bytes
+  continuation.unshift(0x7a_u8, continuation.size.to_u8)
+
+  slice = continuation.to_unsafe.to_slice(continuation.size)
+  continuation = Base64.urlsafe_encode(slice)
+  continuation = URI.escape(continuation)
+  continuation = continuation.bytes
+  continuation.unshift(continuation.size.to_u8)
+
+  continuation.unshift(ucid.size.to_u8)
+  continuation = ucid.bytes + continuation
+  continuation.unshift(0x12.to_u8, ucid.size.to_u8)
+  continuation.unshift(0xe2_u8, 0xa9_u8, 0x85_u8, 0xb2_u8, 2_u8, continuation.size.to_u8)
+
+  slice = continuation.to_unsafe.to_slice(continuation.size)
+  continuation = Base64.urlsafe_encode(slice)
+  continuation = URI.escape(continuation)
+
+  url = "/browse_ajax?action_continuation=1&continuation=#{continuation}"
+
+  return url
+end
+
+def produce_videos_url(ucid, page = 1)
+  page = "#{page}"
+
+  meta = "\x12\x06videos \x00\x30\x02\x38\x01\x60\x01\x6a\x00\x7a"
+  meta += page.size.to_u8.unsafe_chr
+  meta += page
+  meta += "\xb8\x01\x00"
+
+  meta = Base64.urlsafe_encode(meta)
+  meta = URI.escape(meta)
+
+  continuation = "\x12"
+  continuation += ucid.size.to_u8.unsafe_chr
+  continuation += ucid
+  continuation += "\x1a"
+  continuation += meta.size.to_u8.unsafe_chr
+  continuation += meta
+
+  continuation = continuation.size.to_u8.unsafe_chr + continuation
+  continuation = "\xe2\xa9\x85\xb2\x02" + continuation
+
+  continuation = Base64.urlsafe_encode(continuation)
+  continuation = URI.escape(continuation)
+
+  url = "/browse_ajax?continuation=#{continuation}"
+
+  return url
+end
+
+def read_var_int(bytes)
+  numRead = 0
+  result = 0
+
+  read = bytes[numRead]
+
+  if bytes.size == 1
+    result = bytes[0].to_i32
+  else
+    while ((read & 0b10000000) != 0)
+      read = bytes[numRead].to_u64
+      value = (read & 0b01111111)
+      result |= (value << (7 * numRead))
+
+      numRead += 1
+      if numRead > 5
+        raise "VarInt is too big"
+      end
+    end
+  end
+
+  return result
+end
+
+def write_var_int(value : Int)
+  bytes = [] of UInt8
+  value = value.to_u32
+
+  if value == 0
+    bytes = [0_u8]
+  else
+    while value != 0
+      temp = (value & 0b01111111).to_u8
+      value = value >> 7
+
+      if value != 0
+        temp |= 0b10000000
+      end
+
+      bytes << temp
+    end
+  end
+
+  return bytes
+end
+
+def generate_captcha(key)
+  minute = Random::Secure.rand(12)
+  minute_angle = minute * 30
+  minute = minute * 5
+
+  hour = Random::Secure.rand(12)
+  hour_angle = hour * 30 + minute_angle.to_f / 12
+  if hour == 0
+    hour = 12
+  end
+
+  clock_svg = <<-END_SVG
+  <svg viewBox="0 0 100 100" width="200px">
+  <circle cx="50" cy="50" r="45" fill="#eee" stroke="black" stroke-width="2"></circle>
+  
+  <text x="69"     y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 1</text>
+  <text x="82.909" y="34"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 2</text>
+  <text x="88"     y="53"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 3</text>
+  <text x="82.909" y="72"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 4</text>
+  <text x="69"     y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 5</text>
+  <text x="50"     y="91"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 6</text>
+  <text x="31"     y="85.909" text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 7</text>
+  <text x="17.091" y="72"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 8</text>
+  <text x="12"     y="53"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px"> 9</text>
+  <text x="17.091" y="34"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px">10</text>
+  <text x="31"     y="20.091" text-anchor="middle" fill="black" font-family="Arial" font-size="10px">11</text>
+  <text x="50"     y="15"     text-anchor="middle" fill="black" font-family="Arial" font-size="10px">12</text>
+
+  <circle cx="50" cy="50" r="3" fill="black"></circle>
+  <line id="minute" transform="rotate(#{minute_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="16" fill="black" stroke="black" stroke-width="2"></line>
+  <line id="hour"   transform="rotate(#{hour_angle}, 50, 50)" x1="50" y1="50" x2="50" y2="24" fill="black" stroke="black" stroke-width="2"></line>
+  </svg>
+  END_SVG
+
+  challenge = ""
+  convert = Process.run(%(convert -density 1200 -resize 400x400 -background none svg:- png:-), shell: true,
+    input: IO::Memory.new(clock_svg), output: Process::Redirect::Pipe) do |proc|
+    challenge = proc.output.gets_to_end
+    challenge = Base64.strict_encode(challenge)
+    challenge = "data:image/png;base64,#{challenge}"
+  end
+
+  answer = "#{hour}:#{minute.to_s.rjust(2, '0')}"
+  token = OpenSSL::HMAC.digest(:sha256, key, answer)
+  token = Base64.encode(token)
+
+  return {challenge: challenge, token: token}
+end

src/invidious/helpers/macros.cr

@@ -0,0 +1,18 @@
+macro add_mapping(mapping)
+    def initialize({{*mapping.keys.map { |id| "@#{id}".id }}})
+    end
+
+    def to_a
+        return [{{*mapping.keys.map { |id| "@#{id}".id }}}]
+    end
+
+    DB.mapping({{mapping}})
+end
+
+macro templated(filename)
+    render "src/invidious/views/#{{{filename}}}.ecr", "src/invidious/views/layout.ecr"
+end
+
+macro rendered(filename)
+    render "src/invidious/views/#{{{filename}}}.ecr"
+end

src/invidious/helpers/utils.cr

@@ -0,0 +1,129 @@
+# See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
+def ci_lower_bound(pos, n)
+  if n == 0
+    return 0.0
+  end
+
+  # z value here represents a confidence level of 0.95
+  z = 1.96
+  phat = 1.0*pos/n
+
+  return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n)
+end
+
+def elapsed_text(elapsed)
+  millis = elapsed.total_milliseconds
+  return "#{millis.round(2)}ms" if millis >= 1
+
+  "#{(millis * 1000).round(2)}µs"
+end
+
+def make_client(url)
+  context = OpenSSL::SSL::Context::Client.new
+  context.add_options(
+    OpenSSL::SSL::Options::ALL |
+    OpenSSL::SSL::Options::NO_SSL_V2 |
+    OpenSSL::SSL::Options::NO_SSL_V3
+  )
+  client = HTTP::Client.new(url, context)
+  client.read_timeout = 10.seconds
+  client.connect_timeout = 10.seconds
+  return client
+end
+
+def decode_length_seconds(string)
+  length_seconds = string.split(":").map { |a| a.to_i }
+  length_seconds = [0] * (3 - length_seconds.size) + length_seconds
+  length_seconds = Time::Span.new(length_seconds[0], length_seconds[1], length_seconds[2])
+  length_seconds = length_seconds.total_seconds.to_i
+
+  return length_seconds
+end
+
+def decode_time(string)
+  time = string.try &.to_f?
+
+  if !time
+    hours = /(?<hours>\d+)h/.match(string).try &.["hours"].try &.to_f
+    hours ||= 0
+
+    minutes = /(?<minutes>\d+)m(?!s)/.match(string).try &.["minutes"].try &.to_f
+    minutes ||= 0
+
+    seconds = /(?<seconds>\d+)s/.match(string).try &.["seconds"].try &.to_f
+    seconds ||= 0
+
+    millis = /(?<millis>\d+)ms/.match(string).try &.["millis"].try &.to_f
+    millis ||= 0
+
+    time = hours * 3600 + minutes * 60 + seconds + millis / 1000
+  end
+
+  return time
+end
+
+def decode_date(string : String)
+  # Time matches format "20 hours ago", "40 minutes ago"...
+  date = string.split(" ")[-3, 3]
+  delta = date[0].to_i
+
+  case date[1]
+  when .includes? "minute"
+    delta = delta.minutes
+  when .includes? "hour"
+    delta = delta.hours
+  when .includes? "day"
+    delta = delta.days
+  when .includes? "week"
+    delta = delta.weeks
+  when .includes? "month"
+    delta = delta.months
+  when .includes? "year"
+    delta = delta.years
+  else
+    raise "Could not parse #{string}"
+  end
+
+  return Time.now - delta
+end
+
+def recode_date(time : Time)
+  span = Time.now - time
+
+  if span.total_days > 365.0
+    span = {span.total_days / 365, "year"}
+  elsif span.total_days > 30.0
+    span = {span.total_days / 30, "month"}
+  elsif span.total_days > 7.0
+    span = {span.total_days / 7, "week"}
+  elsif span.total_hours > 24.0
+    span = {span.total_days, "day"}
+  elsif span.total_minutes > 60.0
+    span = {span.total_hours, "hour"}
+  else
+    span = {0, "units"}
+  end
+
+  span = {span[0].to_i, span[1]}
+  if span[0] > 1
+    span = {span[0], span[1] + "s"}
+  end
+
+  return span.join(" ")
+end
+
+def number_with_separator(number)
+  number.to_s.reverse.gsub(/(\d{3})(?=\d)/, "\\1,").reverse
+end
+
+def arg_array(array, start = 1)
+  if array.size == 0
+    args = "NULL"
+  else
+    args = [] of String
+    (start..array.size + start - 1).each { |i| args << "($#{i})" }
+    args = args.join(",")
+  end
+
+  return args
+end