Add related videos and clean up video class

invidious.sql

@@ -1,37 +0,0 @@
--- Table: public.videos
-
--- DROP TABLE public.videos;
-
-CREATE TABLE public.videos
-(
-    last_updated timestamp with time zone,
-    video_id text COLLATE pg_catalog."default" NOT NULL,
-    video_info text COLLATE pg_catalog."default",
-    video_html text COLLATE pg_catalog."default",
-    views bigint,
-    likes integer,
-    dislikes integer,
-    rating double precision,
-    description text COLLATE pg_catalog."default",
-    CONSTRAINT videos_pkey PRIMARY KEY (video_id)
-)
-WITH (
-    OIDS = FALSE
-)
-TABLESPACE pg_default;
-
-ALTER TABLE public.videos
-    OWNER to omar;
-
-GRANT ALL ON TABLE public.videos TO kemal;
-
-GRANT ALL ON TABLE public.videos TO omar;
-
--- Index: videos_video_id_idx
-
--- DROP INDEX public.videos_video_id_idx;
-
-CREATE INDEX videos_video_id_idx
-    ON public.videos USING btree
-    (video_id COLLATE pg_catalog."default")
-    TABLESPACE pg_default;

src/invidious.cr

@@ -5,83 +5,51 @@ require "pg"
 require "xml"
 require "time"
 
+PG_DB   = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
+CONTEXT = OpenSSL::SSL::Context::Client.insecure
+
 macro templated(filename)
   render "src/views/#{{{filename}}}.ecr", "src/views/layout.ecr"
 end
 
 class Video
-  getter last_updated : Time
+  module HTTPParamConverter
-  getter video_id : String
+    def self.from_rs(rs)
-  getter video_info : String
+      HTTP::Params.parse(rs.read(String))
-  getter video_html : String
-  getter views : String
-  getter likes : Int32
-  getter dislikes : Int32
-  getter rating : Float64
-  getter description : String
-
-  def initialize(last_updated, video_id, video_info, video_html, views, likes, dislikes, rating, description)
-    @last_updated = last_updated
-    @video_id = video_id
-    @video_info = video_info
-    @video_html = video_html
-    @views = views
-    @likes = likes
-    @dislikes = dislikes
-    @rating = rating
-    @description = description
     end
-
-  def to_a
-    return [@last_updated, @video_id, @video_info, @video_html, @views, @likes, @dislikes, @rating, @description]
   end
 
-  DB.mapping({
+  module XMLConverter
-    last_updated: Time,
+    def self.from_rs(rs)
-    video_id:     String,
+      XML.parse(rs.read(String))
-    video_info:   String,
-    video_html:   String,
-    views:        Int64,
-    likes:        Int32,
-    dislikes:     Int32,
-    rating:       Float64,
-    description:  String,
-  })
     end
-
-def get_video(video_id, context)
-  client = HTTP::Client.new("www.youtube.com", 443, context)
-  video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
-  info = HTTP::Params.parse(video_info)
-  video_html = client.get("/watch?v=#{video_id}").body
-  html = XML.parse(video_html)
-  views = info["view_count"].to_i64
-  rating = info["avg_rating"].to_f64
-
-  likes = html.xpath_node(%q(//button[@title="I like this"]/span))
-  if likes
-    likes = likes.content.delete(",").to_i
-  else
-    likes = 1
   end
 
-  dislikes = html.xpath_node(%q(//button[@title="I dislike this"]/span))
+  def initialize(id, info, html, updated)
-  if dislikes
+    @id = id
-    dislikes = dislikes.content.delete(",").to_i
+    @info = info
-  else
+    @html = html
-    dislikes = 1
+    @updated = updated
   end
 
-  description = html.xpath_node(%q(//p[@id="eow-description"]))
+  def to_a
-  if description
+    return [@id, @info, @html, @updated]
-    description = description.to_xml
-  else
-    description = ""
   end
 
-  video_record = Video.new(Time.now, video_id, video_info, video_html, views, likes, dislikes, rating, description)
+  DB.mapping({
-
+    id:   String,
-  return video_record
+    info: {
+      type:      HTTP::Params,
+      default:   HTTP::Params.parse(""),
+      converter: Video::HTTPParamConverter,
+    },
+    html: {
+      type:      XML::Node,
+      default:   XML.parse(""),
+      converter: Video::XMLConverter,
+    },
+    updated: Time,
+  })
 end
 
 # See http://www.evanmiller.org/how-not-to-sort-by-average-rating.html
@@ -97,73 +65,115 @@ def ci_lower_bound(pos, n)
   return (phat + z*z/(2*n) - z * Math.sqrt((phat*(1 - phat) + z*z/(4*n))/n))/(1 + z*z/n)
 end
 
-get "/" do |env|
+def fetch_video(id)
-  templated "index"
+  client = HTTP::Client.new("www.youtube.com", 443, CONTEXT)
-end
+  info = client.get("/get_video_info?video_id=#{id}&el=info&ps=default&eurl=&gl=US&hl=en").body
+  info = HTTP::Params.parse(info)
 
-pg = DB.open "postgres://kemal:kemal@localhost:5432/invidious"
+  html = client.get("/watch?v=#{id}").body
-context = OpenSSL::SSL::Context::Client.insecure
+  html = XML.parse(html)
 
-get "/watch" do |env|
+  if info["reason"]?
-  video_id = env.params.query["v"]
+    raise info["reason"]
+  end
 
-  if env.params.query["listen"]? && env.params.query["listen"] == "true"
+  video = Video.new(id, info, html, Time.now)
-    env.request.query_params.delete_all("listen")
+
-    listen = true
+  return video
-  else
-    env.request.query_params["listen"] = "true"
-    listen = false
 end
 
-  if pg.query_one?("select exists (select true from videos where video_id = $1)", video_id, as: Bool)
+def get_video(id)
-    video_record = pg.query_one("select * from videos where video_id = $1", video_id, as: Video)
+  if PG_DB.query_one?("SELECT EXISTS ( SELECT true FROM videos WHERE id = $1)", id, as: Bool)
+    video = PG_DB.query_one("SELECT * FROM videos WHERE id = $1", id, as: Video)
 
     # If record was last updated more than 5 hours ago, refresh (expire param in response lasts for 6 hours)
-    if Time.now - video_record.last_updated > Time::Span.new(0, 5, 0, 0)
+    if Time.now - video.updated > Time::Span.new(0, 5, 0, 0)
-      video_record = get_video(video_id, context)
+      video = fetch_video(id)
-      pg.exec("update videos set last_updated = $1, video_info = $3, video_html = $4,\
-      views = $5, likes = $6, dislikes = $7, rating = $8, description = $9 where video_id = $2",
-        video_record.to_a)
     end
   else
-    client = HTTP::Client.new("www.youtube.com", 443, context)
+    video = fetch_video(id)
-    video_info = client.get("/get_video_info?video_id=#{video_id}&el=info&ps=default&eurl=&gl=US&hl=en").body
+    PG_DB.exec("INSERT INTO videos VALUES ($1, $2, $3, $4)", video.to_a)
-    info = HTTP::Params.parse(video_info)
+  end
 
-    if info["reason"]?
+  return video
-      error_message = info["reason"]
-      next templated "error"
 end
 
-    video_record = get_video(video_id, context)
+get "/" do |env|
-    pg.exec("insert into videos values ($1,$2,$3,$4,$5,$6,$7,$8, $9)", video_record.to_a)
+  templated "index"
 end
 
-  # last_updated, video_id, video_info, video_html, views, likes, dislikes, rating
+get "/watch" do |env|
-  video_info = HTTP::Params.parse(video_record.video_info)
+  id = env.params.query["v"]
-  video_html = XML.parse(video_record.video_html)
+
+  begin
+    video = get_video(id)
+  rescue ex
+    error_message = ex.message
+    next templated "error"
+  end
+
+  if env.params.query["listen"]? && env.params.query["listen"] == "true"
+    env.request.query_params.delete_all("listen")
+    listen = true
+  else
+    env.request.query_params["listen"] = "true"
+    listen = false
+  end
 
   fmt_stream = [] of HTTP::Params
-  video_info["url_encoded_fmt_stream_map"].split(",") do |string|
+  video.info["url_encoded_fmt_stream_map"].split(",") do |string|
     fmt_stream << HTTP::Params.parse(string)
   end
 
+  fmt_stream.reverse! # We want lowest quality first
+
   adaptive_fmts = [] of HTTP::Params
-  video_info["adaptive_fmts"].split(",") do |string|
+  video.info["adaptive_fmts"].split(",") do |string|
     adaptive_fmts << HTTP::Params.parse(string)
   end
 
-  fmt_stream.reverse! # We want lowest quality first
+  related_videos = video.html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href))
+  if related_videos.empty?
+    related_videos = video.html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href))
+  end
 
-  related_videos = video_html.xpath_nodes(%q(//li/div/a[contains(@class,"content-link")]/@href))
+  related_videos_list = [] of Video
+  related_videos.each do |related_video|
+    related_id = related_video.content.split("=")[1]
+    begin
+      related_videos_list << get_video(related_id)
+    rescue ex
+      p "#{related_id}: #{ex.message}"
+    end
+  end
 
-  if related_videos.empty?
+  likes = video.html.xpath_node(%q(//button[@title="I like this"]/span))
-    related_videos = video_html.xpath_nodes(%q(//ytd-compact-video-renderer/div/a/@href))
+  if likes
+    likes = likes.content.delete(",").to_i
+  else
+    likes = 1
+  end
+
+  dislikes = video.html.xpath_node(%q(//button[@title="I dislike this"]/span))
+  if dislikes
+    dislikes = dislikes.content.delete(",").to_i
+  else
+    dislikes = 1
   end
 
-  likes = video_record.likes.to_f
+  description = video.html.xpath_node(%q(//p[@id="eow-description"]))
-  dislikes = video_record.dislikes.to_f
+  if description
-  views = video_record.views.to_f
+    description = description.to_xml
+  else
+    description = ""
+  end
+
+  views = video.info["view_count"].to_i64
+  rating = video.info["avg_rating"].to_f64
+
+  likes = likes.to_f
+  dislikes = dislikes.to_f
+  views = views.to_f
 
   engagement = ((dislikes + likes)/views * 100)
   calculated_rating = (likes/(likes + dislikes) * 4 + 1)
@@ -173,9 +183,9 @@ end
 
 get "/search" do |env|
   query = URI.escape(env.params.query["q"])
-  client = HTTP::Client.new("www.youtube.com", 443, context)
+  client = HTTP::Client.new("www.youtube.com", 443, CONTEXT)
-  results_html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body
+  html = client.get("https://www.youtube.com/results?q=#{query}&page=1").body
-  html = XML.parse(results_html)
+  html = XML.parse(html)
 
   videos = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-video-renderer))
   channels = html.xpath_nodes(%q(//div[@class="style-scope ytd-item-section-renderer"]/ytd-channel-renderer))
@@ -185,6 +195,16 @@ get "/search" do |env|
     channels = html.xpath_nodes(%q(//div[contains(@class,"yt-lockup-channel")]/div/div[contains(@class,"yt-lockup-thumbnail")]/a/@href))
   end
 
+  videos_list = [] of Video
+  videos.each do |video|
+    id = video.content.split("=")[1]
+    begin
+      videos_list << get_video(id)
+    rescue ex
+      p "#{id}: #{ex.message}"
+    end
+  end
+
   templated "search"
 end
 

src/views/search.ecr

@@ -1,3 +1,3 @@
-<% videos.each do |video| %>
+<% videos_list.each do |video| %>
-<p><a href="<%= video.content %>"><%= video.content %></a></p>
+<p><a href="/watch?v=<%= video.id %>"><%= video.info["title"] %></a></p>
 <% end %>

src/views/watch.ecr

@@ -1,5 +1,5 @@
-<title><%= video_info["title"] %> - Invidious</title>
+<title><%= video.info["title"] %> - Invidious</title>
-<video style="width: 100%" poster="<%= video_info.has_key?("iurlhq720") ? video_info["iurlhq720"] : video_info["iurlmq"] %>" controls>
+<video style="width: 100%" poster="<%= video.info.has_key?("iurlhq720") ? video.info["iurlhq720"] : video.info["iurlmq"] %>" controls>
     <% if listen %>
         <% adaptive_fmts.each do |fmt| %>
             <% url = fmt["url"] %>
@@ -14,28 +14,29 @@
         <% end %>
     <% end %>
     </video>
-    <h1><%= video_info["title"] %> <a href="/watch?<%= env.request.query %>">
+    <h1><%= video.info["title"] %> <a href="/watch?<%= env.request.query %>">
             <i class="fa <%= listen ? "fa-video-camera" : "fa-volume-up" %>" aria-hidden="true"></i>
         </a>
     </h1>
     <div class="pure-g">
         <div class="pure-u-1 pure-u-md-1-5">
-            <p><i class="fa fa-eye" aria-hidden="true"></i> <%= video_record.views %></p>
+            <p><i class="fa fa-eye" aria-hidden="true"></i> <%= views.to_i %></p>
-            <p><i class="fa fa-thumbs-up" aria-hidden="true"></i> <%= video_record.likes %></p>
+            <p><i class="fa fa-thumbs-up" aria-hidden="true"></i> <%= likes.to_i %></p>
-            <p><i class="fa fa-thumbs-down" aria-hidden="true"></i> <%= video_record.dislikes %></p>
+            <p><i class="fa fa-thumbs-down" aria-hidden="true"></i> <%= dislikes.to_i %></p>
-            <p>Wilson Score : <%= ci_lower_bound(video_record.likes, video_record.likes + video_record.dislikes).round(4) %></p>
+            <p>Wilson Score : <%= ci_lower_bound(likes, likes + dislikes).round(4) %></p>
-            <p>Rating : <%= video_record.rating.round(4) %> / 5</p>
+            <p>Rating : <%= rating.round(4) %> / 5</p>
             <!-- <p>Calculated Rating : <%= calculated_rating.round(4) %> / 5</p> -->
             <p>Engagement : <%= engagement.round(2) %>%</p>
-            <p>Earnings : <%= video_info.has_key?("allowed_ads") ? "~$" + ((video_record.views.to_f / 500).round(2)).to_s : "Unmonetized" %></p>
+            <p>Earnings : <%= video.info.has_key?("allowed_ads") ? "~$" + ((views.to_f / 500).round(2)).to_s : "Unmonetized" %></p>
-            <p>Allowed ads : <br><%= video_info.has_key?("allowed_ads") ? video_info["allowed_ads"] : "Unmonetized" %></p> 
+            <p>Allowed ads : <br><%= video.info.has_key?("allowed_ads") ? video.info["allowed_ads"] : "Unmonetized" %></p> 
         </div>
         <div class="pure-u-1 pure-u-md-3-5">
-            <p><%= video_record.description %></p>
+            <p><%= video.info["author"] %></p>
+            <p><%= description %></p>
         </div>
         <div class="pure-u-1 pure-u-md-1-5">
-            <% related_videos.each do |video| %>
+            <% related_videos_list.each do |video| %>
-            <p><a href="<%= video.content %>"><%= video.content %></a></p>
+            <p><a href="/watch?v=<%= video.id %>"><%= video.info["title"] %></a></p>
             <% end %>
     </div>
 </div>

videos.sql

@@ -0,0 +1,32 @@
+-- Table: public.videos
+
+-- DROP TABLE public.videos;
+
+CREATE TABLE public.videos
+(
+    id text COLLATE pg_catalog."default" NOT NULL,
+    info text COLLATE pg_catalog."default",
+    html text COLLATE pg_catalog."default",
+    updated timestamp with time zone,
+    CONSTRAINT videos_pkey PRIMARY KEY (id)
+)
+WITH (
+    OIDS = FALSE
+)
+TABLESPACE pg_default;
+
+ALTER TABLE public.videos
+    OWNER to omar;
+
+GRANT ALL ON TABLE public.videos TO kemal;
+
+GRANT ALL ON TABLE public.videos TO omar;
+
+-- Index: id_idx
+
+-- DROP INDEX public.id_idx;
+
+CREATE UNIQUE INDEX id_idx
+    ON public.videos USING btree
+    (id COLLATE pg_catalog."default")
+    TABLESPACE pg_default;