From fe8ed6b41bfeb819750719e083da2db3bee211e2 Mon Sep 17 00:00:00 2001
From: Samantaz Fox <coding@samantaz.fr>
Date: Sun, 23 Oct 2022 14:15:16 +0200
Subject: [PATCH] Cache: Create the base of the caching subsystem

---
 config/config.example.yml                  | 15 +++++
 src/invidious/cache.cr                     | 32 ++++++++++
 src/invidious/cache/cacheable_item.cr      |  9 +++
 src/invidious/cache/item_store.cr          | 22 +++++++
 src/invidious/cache/null_item_store.cr     | 24 ++++++++
 src/invidious/cache/postgres_item_store.cr | 70 ++++++++++++++++++++++
 src/invidious/cache/redis_item_store.cr    | 36 +++++++++++
 src/invidious/cache/store_type.cr          |  7 +++
 src/invidious/config.cr                    | 12 ++--
 src/invidious/config/cache.cr              | 14 +++++
 10 files changed, 233 insertions(+), 8 deletions(-)
 create mode 100644 src/invidious/cache.cr
 create mode 100644 src/invidious/cache/cacheable_item.cr
 create mode 100644 src/invidious/cache/item_store.cr
 create mode 100644 src/invidious/cache/null_item_store.cr
 create mode 100644 src/invidious/cache/postgres_item_store.cr
 create mode 100644 src/invidious/cache/redis_item_store.cr
 create mode 100644 src/invidious/cache/store_type.cr
 create mode 100644 src/invidious/config/cache.cr

diff --git a/config/config.example.yml b/config/config.example.yml
index c591eb6a..1fde7ba7 100644
--- a/config/config.example.yml
+++ b/config/config.example.yml
@@ -42,6 +42,21 @@ db:
 
 
 
+#########################################
+#
+#  Cache configuration
+#
+#########################################
+
+cache:
+  ##
+  ## URL of the caching server. To not use a caching server,
+  ## set to an empty string.
+  ##
+  url: "redis://"
+
+
+
 #########################################
 #
 #  Server config
diff --git a/src/invidious/cache.cr b/src/invidious/cache.cr
new file mode 100644
index 00000000..3e532b22
--- /dev/null
+++ b/src/invidious/cache.cr
@@ -0,0 +1,32 @@
+require "./cache/*"
+
+module Invidious::Cache
+  extend self
+
+  INSTANCE = self.init(CONFIG.cache)
+
+  def init(cfg : Config::CacheConfig) : ItemStore
+    # Environment variable takes precedence over local config
+    url = ENV.get?("INVIDIOUS__CACHE__URL").try { |u| URI.parse(u) }
+    url ||= CONFIG.cache.url
+
+    # Determine cache type from URL scheme
+    type = StoreType.parse?(url.scheme || "none") || StoreType::None
+
+    case type
+    when .none?
+      return NullItemStore.new
+    when .postgres?
+      # Use the database URL as a compatibility fallback
+      url ||= CONFIG.database_url
+      return PostgresItemStore.new(url)
+    when .redis?
+      if url.nil?
+        raise InvalidConfigException.new "Redis cache requires an URL."
+      end
+      return RedisItemStore.new(url)
+    else
+      raise InvalidConfigException.new "Invalid cache url. Supported values are redis://, postgres:// or nothing."
+    end
+  end
+end
diff --git a/src/invidious/cache/cacheable_item.cr b/src/invidious/cache/cacheable_item.cr
new file mode 100644
index 00000000..c1295a4a
--- /dev/null
+++ b/src/invidious/cache/cacheable_item.cr
@@ -0,0 +1,9 @@
+require "json"
+
+module Invidious::Cache
+  # Including this module allows the includer object to be cached.
+  # The object will automatically inherit from JSON::Serializable.
+  module CacheableItem
+    include JSON::Serializable
+  end
+end
diff --git a/src/invidious/cache/item_store.cr b/src/invidious/cache/item_store.cr
new file mode 100644
index 00000000..e4ec1201
--- /dev/null
+++ b/src/invidious/cache/item_store.cr
@@ -0,0 +1,22 @@
+require "./cacheable_item"
+
+module Invidious::Cache
+  # Abstract class from which any cached element should inherit
+  # Note: class is used here, instead of a module, in order to benefit
+  # from various compiler checks (e.g methods must be implemented)
+  abstract class ItemStore
+    # Retrieves an item from the store
+    # Returns nil if item wasn't found or is expired
+    abstract def fetch(key : String, *, as : T.class)
+
+    # Stores a given item into cache
+    abstract def store(key : String, value : CacheableItem, expires : Time::Span)
+
+    # Prematurely deletes item(s) from the cache
+    abstract def delete(key : String)
+    abstract def delete(keys : Array(String))
+
+    # Removes all the items stored in the cache
+    abstract def clear
+  end
+end
diff --git a/src/invidious/cache/null_item_store.cr b/src/invidious/cache/null_item_store.cr
new file mode 100644
index 00000000..c26c0804
--- /dev/null
+++ b/src/invidious/cache/null_item_store.cr
@@ -0,0 +1,24 @@
+require "./item_store"
+
+module Invidious::Cache
+  class NullItemStore < ItemStore
+    def initialize
+    end
+
+    def fetch(key : String, *, as : T.class) : T? forall T
+      return nil
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+    end
+
+    def delete(key : String)
+    end
+
+    def delete(keys : Array(String))
+    end
+
+    def clear
+    end
+  end
+end
diff --git a/src/invidious/cache/postgres_item_store.cr b/src/invidious/cache/postgres_item_store.cr
new file mode 100644
index 00000000..cfbe52e2
--- /dev/null
+++ b/src/invidious/cache/postgres_item_store.cr
@@ -0,0 +1,70 @@
+require "./item_store"
+require "json"
+require "pg"
+
+module Invidious::Cache
+  class PostgresItemStore < ItemStore
+    @db : DB::Database
+    @node_name : String
+
+    def initialize(url : URI, @node_name = "")
+      @db = DB.open url
+    end
+
+    def fetch(key : String, *, as : T.class) : T? forall T
+      request = <<-SQL
+        SELECT info,updated
+        FROM videos
+        WHERE id = $1
+      SQL
+
+      value, expires = @db.query_one?(request, key, as: {String?, Time?})
+
+      if expires < Time.utc
+        self.delete(key)
+        return nil
+      else
+        return T.from_json(JSON::PullParser.new(value))
+      end
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+      request = <<-SQL
+        INSERT INTO videos
+        VALUES ($1, $2, $3)
+        ON CONFLICT (id) DO
+          UPDATE
+          SET info = $2, updated = $3
+      SQL
+
+      @db.exec(request, key, value.to_json, Time.utc + expires)
+    end
+
+    def delete(key : String)
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE id = $1
+      SQL
+
+      @db.exec(request, key)
+    end
+
+    def delete(keys : Array(String))
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE id = ANY($1::TEXT[])
+      SQL
+
+      @db.exec(request, keys)
+    end
+
+    def clear
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE updated < now()
+      SQL
+
+      @db.exec(request)
+    end
+  end
+end
diff --git a/src/invidious/cache/redis_item_store.cr b/src/invidious/cache/redis_item_store.cr
new file mode 100644
index 00000000..ccf847a6
--- /dev/null
+++ b/src/invidious/cache/redis_item_store.cr
@@ -0,0 +1,36 @@
+require "./item_store"
+require "json"
+require "redis"
+
+module Invidious::Cache
+  class RedisItemStore < ItemStore
+    @redis : Redis::PooledClient
+    @node_name : String
+
+    def initialize(url : URI, @node_name = "")
+      @redis = Redis::PooledClient.new url
+    end
+
+    def fetch(key : String, *, as : T.class) : (T | Nil) forall T
+      value = @redis.get(key)
+      return nil if value.nil?
+      return T.from_json(JSON::PullParser.new(value))
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+      @redis.set(key, value, ex: expires.to_i)
+    end
+
+    def delete(key : String)
+      @redis.del(key)
+    end
+
+    def delete(keys : Array(String))
+      @redis.del(keys)
+    end
+
+    def clear
+      @redis.flushdb
+    end
+  end
+end
diff --git a/src/invidious/cache/store_type.cr b/src/invidious/cache/store_type.cr
new file mode 100644
index 00000000..32f84069
--- /dev/null
+++ b/src/invidious/cache/store_type.cr
@@ -0,0 +1,7 @@
+module Invidious::Cache
+  enum StoreType
+    None
+    Postgres
+    Redis
+  end
+end
diff --git a/src/invidious/config.cr b/src/invidious/config.cr
index 2ade568c..f180cf24 100644
--- a/src/invidious/config.cr
+++ b/src/invidious/config.cr
@@ -74,6 +74,8 @@ class Config
 
   # Jobs config structure. See jobs.cr and jobs/base_job.cr
   property jobs = Invidious::Jobs::JobsConfig.new
+  # Cache configuration. See cache/cache.cr
+  property cache = Invidious::Config::CacheConfig.new
 
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
@@ -201,14 +203,8 @@ class Config
     # Build database_url from db.* if it's not set directly
     if config.database_url.to_s.empty?
       if db = config.db
-        config.database_url = URI.new(
-          scheme: "postgres",
-          user: db.user,
-          password: db.password,
-          host: db.host,
-          port: db.port,
-          path: db.dbname,
-        )
+        db.scheme = "postgres"
+        config.database_url = db.to_uri
       else
         puts "Config : Either database_url or db.* is required"
         exit(1)
diff --git a/src/invidious/config/cache.cr b/src/invidious/config/cache.cr
new file mode 100644
index 00000000..3fe00e26
--- /dev/null
+++ b/src/invidious/config/cache.cr
@@ -0,0 +1,14 @@
+require "../cache/store_type"
+
+module Invidious::Config
+  struct CacheConfig
+    include YAML::Serializable
+
+    @[YAML::Field(converter: IV::Config::URIConverter)]
+    @url : URI? = URI.parse("")
+
+    # Required because of YAML serialization
+    def initialize
+    end
+  end
+end