Implementing in memory response cache

Cargo.lock

@@ -146,6 +146,7 @@ dependencies = [
  "notify",
  "noyalib",
  "pingora",
+ "pingora-cache",
  "pingora-core",
  "pingora-http",
  "pingora-limits",

Cargo.toml

@@ -21,6 +21,7 @@ pingora-core = "0.8.1"
 pingora-proxy = "0.8.1"
 pingora-http = "0.8.1"
 pingora-limits = "0.8.1"
+pingora-cache = "0.8.1"
 dashmap = "7.0.0-rc2"
 async-trait = "0.1.89"
 log = "0.4.33"
@@ -53,3 +54,4 @@ mimalloc = { version = "0.1.52", default-features = false }
 signal-hook = "0.4.4"
 sd-notify = "0.5.0"
 libc = "0.2.186"
+

src/utils/lazylock.rs

@@ -1,7 +1,11 @@
 use dashmap::DashMap;
 use moka::sync::Cache;
+use pingora_cache::eviction::simple_lru::Manager;
+use pingora_cache::lock::CacheLock;
+use pingora_cache::MemCache;
 use pingora_limits::rate::Rate;
 use std::net::IpAddr;
+use std::sync::OnceLock;
 use std::sync::{Arc, LazyLock};
 use std::time::Duration;
 
@@ -9,3 +13,7 @@ pub static REVERSE_STORE: LazyLock<DashMap<String, String>> = LazyLock::new(Dash
 pub static RATE_LIMITER: LazyLock<Rate> = LazyLock::new(|| Rate::new(Duration::from_secs(1)));
 pub static REQUESTS_4XX: LazyLock<Cache<IpAddr, u32>> = LazyLock::new(|| Cache::builder().time_to_live(Duration::from_secs(1)).build());
 pub static LOCALHOST: LazyLock<Arc<str>> = LazyLock::new(|| Arc::from("localhost"));
+pub static MEM_CACHE: LazyLock<MemCache> = LazyLock::new(MemCache::new);
+pub static CACHE_LOCK: LazyLock<CacheLock> = LazyLock::new(|| CacheLock::new(Duration::from_secs(30)));
+pub static EVICTION: OnceLock<Manager> = OnceLock::new();
+pub static CACHE_TTL: OnceLock<u64> = OnceLock::new();

src/utils/metrics.rs

@@ -1,3 +1,5 @@
+use crate::utils::lazylock::EVICTION;
+use pingora_cache::eviction::EvictionManager;
 use pingora_http::Method;
 use pingora_http::StatusCode;
 use pingora_http::Version;
@@ -20,6 +22,11 @@ pub static MEMORY_USAGE: LazyLock<IntGauge> = LazyLock::new(|| register_int_gaug
 pub static ACTIVE_SESSIONS: LazyLock<IntGauge> = LazyLock::new(|| register_int_gauge!("aralez_active_sessions", "Current number of active sessions").unwrap());
 pub static REQUEST_COUNT: LazyLock<IntCounter> = LazyLock::new(|| register_int_counter!("aralez_requests_total", "Total number of requests handled by Aralez").unwrap());
 
+pub static CACHE_SIZE_BYTES: LazyLock<IntGauge> = LazyLock::new(|| register_int_gauge!("aralez_cache_size_bytes", "Current cache size in bytes").unwrap());
+pub static CACHE_ITEMS: LazyLock<IntGauge> = LazyLock::new(|| register_int_gauge!("aralez_cache_items", "Current number of cached objects").unwrap());
+pub static CACHE_EVICTED_BYTES: LazyLock<IntGauge> = LazyLock::new(|| register_int_gauge!("aralez_cache_evicted_bytes_total", "Total bytes evicted from cache").unwrap());
+
+pub static CACHE_EVICTED_ITEMS: LazyLock<IntGauge> = LazyLock::new(|| register_int_gauge!("aralez_cache_evicted_items_total", "Total cache items evicted").unwrap());
 pub static RESPONSE_CODES: LazyLock<IntCounterVec> =
     LazyLock::new(|| register_int_counter_vec!("aralez_responses_total", "Responses grouped by status code", &["status"]).unwrap());
 
@@ -55,6 +62,13 @@ pub fn calc_metrics(metric_types: &MetricTypes) {
     REQUESTS_BY_METHOD.with_label_values(&[metric_types.method.as_str()]).inc();
     REQUESTS_BY_UPSTREAM.with_label_values(&[metric_types.upstream.as_ref()]).inc();
     RESPONSE_LATENCY.observe(metric_types.latency.as_secs_f64());
+
+    if let Some(eviction) = EVICTION.get() {
+        CACHE_SIZE_BYTES.set(eviction.total_size() as i64);
+        CACHE_ITEMS.set(eviction.total_items() as i64);
+        CACHE_EVICTED_BYTES.set(eviction.evicted_size() as i64);
+        CACHE_EVICTED_ITEMS.set(eviction.evicted_items() as i64);
+    }
 }
 
 pub(crate) fn get_memory_usage() -> usize {

src/utils/structs.rs

@@ -110,6 +110,8 @@ pub struct AppConfig {
     pub hc_method: String,
     pub upstreams_conf: String,
     pub log_level: String,
+    pub cache_size_mb: Option<usize>,
+    pub cache_ttl: Option<u64>,
     pub access_log: Option<String>,
     pub pid_file: Option<String>,
     pub master_key: Option<String>,

src/web/logging.rs

@@ -10,6 +10,7 @@ use log4rs::{
     config::{Appender, Config as Log4rsConfig, Root},
     encode::pattern::PatternEncoder,
 };
+use pingora_cache::CachePhase;
 use pingora_http::Version;
 use pingora_proxy::Session;
 use std::net::{IpAddr, Ipv4Addr};
@@ -23,6 +24,7 @@ pub struct LogMessage {
     pub client_ip: IpAddr,
     pub version: Version,
     pub user_agent: String,
+    pub cache_status: CachePhase,
 }
 static LOG_SENDER: OnceLock<mpsc::Sender<LogMessage>> = OnceLock::new();
 static ACCESS_LOG: OnceLock<LogLevel> = OnceLock::new();
@@ -126,13 +128,19 @@ pub fn access_log(response_code: u16, summary: &str, session: &Session) {
         .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST));
 
     let user_agent = session.req_header().headers.get("user-agent").and_then(|v| v.to_str().ok()).unwrap_or("-");
-
+    // let cache_status = match session.cache.phase() {
+    //     CachePhase::Hit => "Hit",
+    //     CachePhase::Miss => "Miss",
+    //     CachePhase::Expired => "Expired",
+    //     _ => "Disabled",
+    // };
     let log = LogMessage {
         response_code,
         summary: summary.to_owned(),
         client_ip: ip,
         version: session.req_header().version,
         user_agent: user_agent.to_owned(),
+        cache_status: session.cache.phase(),
     };
 
     if let Some(sender) = LOG_SENDER.get() {
@@ -156,8 +164,13 @@ pub fn init_logging(enabled: Option<String>) {
 pub fn log_receiver(mut receiver: mpsc::Receiver<LogMessage>) {
     while let Some(msg) = receiver.blocking_recv() {
         info!(
-            "{}, {}, client: {}, version: {:?}, useragent: {}",
-            msg.response_code, msg.summary, msg.client_ip, msg.version, msg.user_agent,
+            "{}, {}, {}, client: {}, version: {:?}, useragent: {}",
+            msg.response_code,
+            msg.cache_status.as_str(),
+            msg.summary,
+            msg.client_ip,
+            msg.version,
+            msg.user_agent,
         );
     }
 }

src/web/proxyhttp.rs

@@ -1,5 +1,5 @@
 use crate::utils::auth::authenticate;
-use crate::utils::lazylock::{LOCALHOST, RATE_LIMITER, REQUESTS_4XX, REVERSE_STORE};
+use crate::utils::lazylock::{CACHE_LOCK, CACHE_TTL, EVICTION, LOCALHOST, MEM_CACHE, RATE_LIMITER, REQUESTS_4XX, REVERSE_STORE};
 use crate::utils::metrics::*;
 use crate::utils::structs::{AppConfig, Extraparams, Headers, InnerMap, UpstreamsDashMap, UpstreamsIdMap};
 use crate::web::gethosts::{GetHost, GetHostsReturHeaders};
@@ -8,9 +8,11 @@ use arc_swap::ArcSwap;
 use async_trait::async_trait;
 use axum::body::Bytes;
 use log::error;
-use pingora::http::{RequestHeader, ResponseHeader, StatusCode};
+use pingora::http::{Method, RequestHeader, ResponseHeader, StatusCode};
 use pingora::prelude::*;
 use pingora::ErrorSource::Upstream;
+use pingora_cache::cache_control::CacheControl;
+use pingora_cache::{CacheKey, CacheMetaDefaults, RespCacheable};
 use pingora_core::listeners::ALPN;
 use pingora_core::prelude::HttpPeer;
 use pingora_proxy::{ProxyHttp, Session};
@@ -18,10 +20,10 @@ use sha2::{Digest, Sha256};
 use std::cell::RefCell;
 use std::fmt::Write;
 use std::sync::Arc;
+use std::time::Duration;
 use tokio::time::Instant;
 
 thread_local! {static IP_BUFFER: RefCell<String> = RefCell::new(String::with_capacity(50));}
-
 #[derive(Clone)]
 pub struct LB {
     pub ump_upst: Arc<UpstreamsDashMap>,
@@ -31,6 +33,7 @@ pub struct LB {
     pub server_headers: Arc<Headers>,
     pub config: Arc<AppConfig>,
     pub extraparams: Arc<ArcSwap<Extraparams>>,
+    pub cache_enabled: bool,
 }
 
 pub struct Context {
@@ -57,6 +60,34 @@ impl ProxyHttp for LB {
             x4xx_limit: None,
         }
     }
+
+    fn cache_key_callback(&self, session: &Session, _ctx: &mut Self::CTX) -> Result<CacheKey> {
+        Ok(CacheKey::new(_ctx.hostname.clone().unwrap_or_default().as_ref(), session.req_header().uri.to_string(), ""))
+    }
+    fn request_cache_filter(&self, session: &mut Session, _ctx: &mut Self::CTX) -> Result<()> {
+        if self.cache_enabled && session.req_header().method == Method::GET {
+            if let Some(eviction) = EVICTION.get() {
+                session.cache.enable(&*MEM_CACHE, Some(eviction), None, Some(&*CACHE_LOCK), None);
+            }
+        }
+        Ok(())
+    }
+    fn response_cache_filter(&self, _session: &Session, resp: &ResponseHeader, _ctx: &mut Self::CTX) -> Result<RespCacheable> {
+        let cc = CacheControl::from_resp_headers(resp);
+        let defaults = CacheMetaDefaults::new(
+            |status| {
+                if status.as_u16() == 200 {
+                    Some(Duration::from_secs(*CACHE_TTL.get().unwrap_or(&60)))
+                } else {
+                    None
+                }
+            },
+            0,
+            0,
+        );
+        Ok(pingora_cache::filters::resp_cacheable(cc.as_ref(), resp.clone(), false, &defaults))
+    }
+
     async fn request_filter(&self, session: &mut Session, _ctx: &mut Self::CTX) -> Result<bool> {
         ACTIVE_SESSIONS.inc();
         let hostname = return_header_host_from_upstream(session, &self.ump_upst);
@@ -159,7 +190,6 @@ impl ProxyHttp for LB {
             Some(hostname) => match ctx.upstream_peer.as_ref() {
                 Some(innermap) => {
                     let mut peer = Box::new(HttpPeer::new((&*innermap.address, innermap.port), innermap.is_ssl, hostname.to_string()));
-
                     if innermap.is_http2 {
                         peer.options.alpn = ALPN::H2;
                     }
@@ -308,6 +338,13 @@ impl ProxyHttp for LB {
                 }
             }
         }
+        // match session.cache.phase() {
+        //     CachePhase::Hit => info!("Cache Status: HIT (Served from Memory)"),
+        //     CachePhase::Miss => info!("Cache Status: MISS (Fetched from Upstream)"),
+        //     CachePhase::Expired => info!("Cache Status: EXPIRED (Revalidating)"),
+        //     _ => {}
+        // }
+
         access_log(response_code, &self.request_summary(session, ctx), session);
     }
 }

src/web/start.rs

@@ -2,6 +2,7 @@
 use crate::tls::grades;
 use crate::tls::load;
 use crate::tls::load::CertificateConfig;
+use crate::utils::lazylock::{CACHE_TTL, EVICTION};
 use crate::utils::structs::Extraparams;
 use crate::utils::tools::*;
 use crate::web::logging::init_access_log;
@@ -10,6 +11,7 @@ use arc_swap::ArcSwap;
 use dashmap::DashMap;
 use log::info;
 use pingora::tls::ssl::{SslAlert, SslRef};
+use pingora_cache::eviction::simple_lru::Manager;
 use pingora_core::listeners::tls::TlsSettings;
 use pingora_core::listeners::TcpSocketOptions;
 use pingora_core::prelude::{background_service, Opt};
@@ -32,6 +34,13 @@ pub fn run() {
     let file = parameters.conf.clone().unwrap();
     let maincfg = crate::utils::parceyaml::parce_main_config(file.as_str());
 
+    let cache_enabled = maincfg.cache_size_mb.is_some();
+    CACHE_TTL.set(maincfg.cache_ttl.unwrap_or(60)).unwrap();
+    if let Some(size) = maincfg.cache_size_mb {
+        let cache_size = size * 1024 * 1024;
+        drop(EVICTION.set(Manager::new(cache_size)));
+    }
+
     let mut server = Server::new(parameters).unwrap();
     server.bootstrap();
 
@@ -59,6 +68,7 @@ pub fn run() {
         client_headers: ch_config,
         server_headers: sh_config,
         extraparams: ec_config,
+        cache_enabled: cache_enabled,
     };
     let al = cfg.access_log.clone().unwrap_or("none".to_string());
     init_access_log(al.as_str());