Cache: Create the base of the caching subsystem

config/config.example.yml

@@ -42,6 +42,21 @@ db:
 
 
 
+#########################################
+#
+#  Cache configuration
+#
+#########################################
+
+cache:
+  ##
+  ## URL of the caching server. To not use a caching server,
+  ## set to an empty string.
+  ##
+  url: "redis://"
+
+
+
 #########################################
 #
 #  Server config

src/invidious/cache.cr

@@ -0,0 +1,32 @@
+require "./cache/*"
+
+module Invidious::Cache
+  extend self
+
+  INSTANCE = self.init(CONFIG.cache)
+
+  def init(cfg : Config::CacheConfig) : ItemStore
+    # Environment variable takes precedence over local config
+    url = ENV.get?("INVIDIOUS__CACHE__URL").try { |u| URI.parse(u) }
+    url ||= CONFIG.cache.url
+
+    # Determine cache type from URL scheme
+    type = StoreType.parse?(url.scheme || "none") || StoreType::None
+
+    case type
+    when .none?
+      return NullItemStore.new
+    when .postgres?
+      # Use the database URL as a compatibility fallback
+      url ||= CONFIG.database_url
+      return PostgresItemStore.new(url)
+    when .redis?
+      if url.nil?
+        raise InvalidConfigException.new "Redis cache requires an URL."
+      end
+      return RedisItemStore.new(url)
+    else
+      raise InvalidConfigException.new "Invalid cache url. Supported values are redis://, postgres:// or nothing."
+    end
+  end
+end

src/invidious/cache/cacheable_item.cr

@@ -0,0 +1,9 @@
+require "json"
+
+module Invidious::Cache
+  # Including this module allows the includer object to be cached.
+  # The object will automatically inherit from JSON::Serializable.
+  module CacheableItem
+    include JSON::Serializable
+  end
+end

src/invidious/cache/item_store.cr

@@ -0,0 +1,22 @@
+require "./cacheable_item"
+
+module Invidious::Cache
+  # Abstract class from which any cached element should inherit
+  # Note: class is used here, instead of a module, in order to benefit
+  # from various compiler checks (e.g methods must be implemented)
+  abstract class ItemStore
+    # Retrieves an item from the store
+    # Returns nil if item wasn't found or is expired
+    abstract def fetch(key : String, *, as : T.class)
+
+    # Stores a given item into cache
+    abstract def store(key : String, value : CacheableItem, expires : Time::Span)
+
+    # Prematurely deletes item(s) from the cache
+    abstract def delete(key : String)
+    abstract def delete(keys : Array(String))
+
+    # Removes all the items stored in the cache
+    abstract def clear
+  end
+end

src/invidious/cache/null_item_store.cr

@@ -0,0 +1,24 @@
+require "./item_store"
+
+module Invidious::Cache
+  class NullItemStore < ItemStore
+    def initialize
+    end
+
+    def fetch(key : String, *, as : T.class) : T? forall T
+      return nil
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+    end
+
+    def delete(key : String)
+    end
+
+    def delete(keys : Array(String))
+    end
+
+    def clear
+    end
+  end
+end

src/invidious/cache/postgres_item_store.cr

@@ -0,0 +1,70 @@
+require "./item_store"
+require "json"
+require "pg"
+
+module Invidious::Cache
+  class PostgresItemStore < ItemStore
+    @db : DB::Database
+    @node_name : String
+
+    def initialize(url : URI, @node_name = "")
+      @db = DB.open url
+    end
+
+    def fetch(key : String, *, as : T.class) : T? forall T
+      request = <<-SQL
+        SELECT info,updated
+        FROM videos
+        WHERE id = $1
+      SQL
+
+      value, expires = @db.query_one?(request, key, as: {String?, Time?})
+
+      if expires < Time.utc
+        self.delete(key)
+        return nil
+      else
+        return T.from_json(JSON::PullParser.new(value))
+      end
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+      request = <<-SQL
+        INSERT INTO videos
+        VALUES ($1, $2, $3)
+        ON CONFLICT (id) DO
+          UPDATE
+          SET info = $2, updated = $3
+      SQL
+
+      @db.exec(request, key, value.to_json, Time.utc + expires)
+    end
+
+    def delete(key : String)
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE id = $1
+      SQL
+
+      @db.exec(request, key)
+    end
+
+    def delete(keys : Array(String))
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE id = ANY($1::TEXT[])
+      SQL
+
+      @db.exec(request, keys)
+    end
+
+    def clear
+      request = <<-SQL
+        DELETE FROM videos *
+        WHERE updated < now()
+      SQL
+
+      @db.exec(request)
+    end
+  end
+end

src/invidious/cache/redis_item_store.cr

@@ -0,0 +1,36 @@
+require "./item_store"
+require "json"
+require "redis"
+
+module Invidious::Cache
+  class RedisItemStore < ItemStore
+    @redis : Redis::PooledClient
+    @node_name : String
+
+    def initialize(url : URI, @node_name = "")
+      @redis = Redis::PooledClient.new url
+    end
+
+    def fetch(key : String, *, as : T.class) : (T | Nil) forall T
+      value = @redis.get(key)
+      return nil if value.nil?
+      return T.from_json(JSON::PullParser.new(value))
+    end
+
+    def store(key : String, value : CacheableItem, expires : Time::Span)
+      @redis.set(key, value, ex: expires.to_i)
+    end
+
+    def delete(key : String)
+      @redis.del(key)
+    end
+
+    def delete(keys : Array(String))
+      @redis.del(keys)
+    end
+
+    def clear
+      @redis.flushdb
+    end
+  end
+end

src/invidious/cache/store_type.cr

@@ -0,0 +1,7 @@
+module Invidious::Cache
+  enum StoreType
+    None
+    Postgres
+    Redis
+  end
+end

src/invidious/config.cr

@@ -74,6 +74,8 @@ class Config
 
   # Jobs config structure. See jobs.cr and jobs/base_job.cr
   property jobs = Invidious::Jobs::JobsConfig.new
+  # Cache configuration. See cache/cache.cr
+  property cache = Invidious::Config::CacheConfig.new
 
   # Used to tell Invidious it is behind a proxy, so links to resources should be https://
   property https_only : Bool?
@@ -201,14 +203,8 @@ class Config
     # Build database_url from db.* if it's not set directly
     if config.database_url.to_s.empty?
       if db = config.db
-        config.database_url = URI.new(
+        db.scheme = "postgres"
-          scheme: "postgres",
+        config.database_url = db.to_uri
-          user: db.user,
-          password: db.password,
-          host: db.host,
-          port: db.port,
-          path: db.dbname,
-        )
       else
         puts "Config : Either database_url or db.* is required"
         exit(1)

src/invidious/config/cache.cr

@@ -0,0 +1,14 @@
+require "../cache/store_type"
+
+module Invidious::Config
+  struct CacheConfig
+    include YAML::Serializable
+
+    @[YAML::Field(converter: IV::Config::URIConverter)]
+    @url : URI? = URI.parse("")
+
+    # Required because of YAML serialization
+    def initialize
+    end
+  end
+end