From b73ccc527f29fa7cf83ac6c1424d3b8b19c33730 Mon Sep 17 00:00:00 2001 From: Ara Sadoyan Date: Mon, 20 Jul 2026 12:25:27 +0200 Subject: [PATCH] Implementing in memory response cache --- Cargo.lock | 1 + Cargo.toml | 2 ++ src/utils/lazylock.rs | 8 ++++++++ src/utils/metrics.rs | 14 ++++++++++++++ src/utils/structs.rs | 2 ++ src/web/logging.rs | 19 +++++++++++++++--- src/web/proxyhttp.rs | 45 +++++++++++++++++++++++++++++++++++++++---- src/web/start.rs | 10 ++++++++++ 8 files changed, 94 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fcd60d4..4c41d8e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -146,6 +146,7 @@ dependencies = [ "notify", "noyalib", "pingora", + "pingora-cache", "pingora-core", "pingora-http", "pingora-limits", diff --git a/Cargo.toml b/Cargo.toml index d42f328..6c31066 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ pingora-core = "0.8.1" pingora-proxy = "0.8.1" pingora-http = "0.8.1" pingora-limits = "0.8.1" +pingora-cache = "0.8.1" dashmap = "7.0.0-rc2" async-trait = "0.1.89" log = "0.4.33" @@ -53,3 +54,4 @@ mimalloc = { version = "0.1.52", default-features = false } signal-hook = "0.4.4" sd-notify = "0.5.0" libc = "0.2.186" + diff --git a/src/utils/lazylock.rs b/src/utils/lazylock.rs index 63676e2..c1c8d00 100644 --- a/src/utils/lazylock.rs +++ b/src/utils/lazylock.rs @@ -1,7 +1,11 @@ use dashmap::DashMap; use moka::sync::Cache; +use pingora_cache::eviction::simple_lru::Manager; +use pingora_cache::lock::CacheLock; +use pingora_cache::MemCache; use pingora_limits::rate::Rate; use std::net::IpAddr; +use std::sync::OnceLock; use std::sync::{Arc, LazyLock}; use std::time::Duration; @@ -9,3 +13,7 @@ pub static REVERSE_STORE: LazyLock> = LazyLock::new(Dash pub static RATE_LIMITER: LazyLock = LazyLock::new(|| Rate::new(Duration::from_secs(1))); pub static REQUESTS_4XX: LazyLock> = LazyLock::new(|| Cache::builder().time_to_live(Duration::from_secs(1)).build()); pub static LOCALHOST: LazyLock> = LazyLock::new(|| Arc::from("localhost")); +pub static MEM_CACHE: LazyLock = LazyLock::new(MemCache::new); +pub static CACHE_LOCK: LazyLock = LazyLock::new(|| CacheLock::new(Duration::from_secs(30))); +pub static EVICTION: OnceLock = OnceLock::new(); +pub static CACHE_TTL: OnceLock = OnceLock::new(); diff --git a/src/utils/metrics.rs b/src/utils/metrics.rs index b0a921e..377071f 100644 --- a/src/utils/metrics.rs +++ b/src/utils/metrics.rs @@ -1,3 +1,5 @@ +use crate::utils::lazylock::EVICTION; +use pingora_cache::eviction::EvictionManager; use pingora_http::Method; use pingora_http::StatusCode; use pingora_http::Version; @@ -20,6 +22,11 @@ pub static MEMORY_USAGE: LazyLock = LazyLock::new(|| register_int_gaug pub static ACTIVE_SESSIONS: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_active_sessions", "Current number of active sessions").unwrap()); pub static REQUEST_COUNT: LazyLock = LazyLock::new(|| register_int_counter!("aralez_requests_total", "Total number of requests handled by Aralez").unwrap()); +pub static CACHE_SIZE_BYTES: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_cache_size_bytes", "Current cache size in bytes").unwrap()); +pub static CACHE_ITEMS: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_cache_items", "Current number of cached objects").unwrap()); +pub static CACHE_EVICTED_BYTES: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_cache_evicted_bytes_total", "Total bytes evicted from cache").unwrap()); + +pub static CACHE_EVICTED_ITEMS: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_cache_evicted_items_total", "Total cache items evicted").unwrap()); pub static RESPONSE_CODES: LazyLock = LazyLock::new(|| register_int_counter_vec!("aralez_responses_total", "Responses grouped by status code", &["status"]).unwrap()); @@ -55,6 +62,13 @@ pub fn calc_metrics(metric_types: &MetricTypes) { REQUESTS_BY_METHOD.with_label_values(&[metric_types.method.as_str()]).inc(); REQUESTS_BY_UPSTREAM.with_label_values(&[metric_types.upstream.as_ref()]).inc(); RESPONSE_LATENCY.observe(metric_types.latency.as_secs_f64()); + + if let Some(eviction) = EVICTION.get() { + CACHE_SIZE_BYTES.set(eviction.total_size() as i64); + CACHE_ITEMS.set(eviction.total_items() as i64); + CACHE_EVICTED_BYTES.set(eviction.evicted_size() as i64); + CACHE_EVICTED_ITEMS.set(eviction.evicted_items() as i64); + } } pub(crate) fn get_memory_usage() -> usize { diff --git a/src/utils/structs.rs b/src/utils/structs.rs index 780f5d6..c369712 100644 --- a/src/utils/structs.rs +++ b/src/utils/structs.rs @@ -110,6 +110,8 @@ pub struct AppConfig { pub hc_method: String, pub upstreams_conf: String, pub log_level: String, + pub cache_size_mb: Option, + pub cache_ttl: Option, pub access_log: Option, pub pid_file: Option, pub master_key: Option, diff --git a/src/web/logging.rs b/src/web/logging.rs index c51babf..6471376 100644 --- a/src/web/logging.rs +++ b/src/web/logging.rs @@ -10,6 +10,7 @@ use log4rs::{ config::{Appender, Config as Log4rsConfig, Root}, encode::pattern::PatternEncoder, }; +use pingora_cache::CachePhase; use pingora_http::Version; use pingora_proxy::Session; use std::net::{IpAddr, Ipv4Addr}; @@ -23,6 +24,7 @@ pub struct LogMessage { pub client_ip: IpAddr, pub version: Version, pub user_agent: String, + pub cache_status: CachePhase, } static LOG_SENDER: OnceLock> = OnceLock::new(); static ACCESS_LOG: OnceLock = OnceLock::new(); @@ -126,13 +128,19 @@ pub fn access_log(response_code: u16, summary: &str, session: &Session) { .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST)); let user_agent = session.req_header().headers.get("user-agent").and_then(|v| v.to_str().ok()).unwrap_or("-"); - + // let cache_status = match session.cache.phase() { + // CachePhase::Hit => "Hit", + // CachePhase::Miss => "Miss", + // CachePhase::Expired => "Expired", + // _ => "Disabled", + // }; let log = LogMessage { response_code, summary: summary.to_owned(), client_ip: ip, version: session.req_header().version, user_agent: user_agent.to_owned(), + cache_status: session.cache.phase(), }; if let Some(sender) = LOG_SENDER.get() { @@ -156,8 +164,13 @@ pub fn init_logging(enabled: Option) { pub fn log_receiver(mut receiver: mpsc::Receiver) { while let Some(msg) = receiver.blocking_recv() { info!( - "{}, {}, client: {}, version: {:?}, useragent: {}", - msg.response_code, msg.summary, msg.client_ip, msg.version, msg.user_agent, + "{}, {}, {}, client: {}, version: {:?}, useragent: {}", + msg.response_code, + msg.cache_status.as_str(), + msg.summary, + msg.client_ip, + msg.version, + msg.user_agent, ); } } diff --git a/src/web/proxyhttp.rs b/src/web/proxyhttp.rs index a2bc60e..76eb7a0 100644 --- a/src/web/proxyhttp.rs +++ b/src/web/proxyhttp.rs @@ -1,5 +1,5 @@ use crate::utils::auth::authenticate; -use crate::utils::lazylock::{LOCALHOST, RATE_LIMITER, REQUESTS_4XX, REVERSE_STORE}; +use crate::utils::lazylock::{CACHE_LOCK, CACHE_TTL, EVICTION, LOCALHOST, MEM_CACHE, RATE_LIMITER, REQUESTS_4XX, REVERSE_STORE}; use crate::utils::metrics::*; use crate::utils::structs::{AppConfig, Extraparams, Headers, InnerMap, UpstreamsDashMap, UpstreamsIdMap}; use crate::web::gethosts::{GetHost, GetHostsReturHeaders}; @@ -8,9 +8,11 @@ use arc_swap::ArcSwap; use async_trait::async_trait; use axum::body::Bytes; use log::error; -use pingora::http::{RequestHeader, ResponseHeader, StatusCode}; +use pingora::http::{Method, RequestHeader, ResponseHeader, StatusCode}; use pingora::prelude::*; use pingora::ErrorSource::Upstream; +use pingora_cache::cache_control::CacheControl; +use pingora_cache::{CacheKey, CacheMetaDefaults, RespCacheable}; use pingora_core::listeners::ALPN; use pingora_core::prelude::HttpPeer; use pingora_proxy::{ProxyHttp, Session}; @@ -18,10 +20,10 @@ use sha2::{Digest, Sha256}; use std::cell::RefCell; use std::fmt::Write; use std::sync::Arc; +use std::time::Duration; use tokio::time::Instant; thread_local! {static IP_BUFFER: RefCell = RefCell::new(String::with_capacity(50));} - #[derive(Clone)] pub struct LB { pub ump_upst: Arc, @@ -31,6 +33,7 @@ pub struct LB { pub server_headers: Arc, pub config: Arc, pub extraparams: Arc>, + pub cache_enabled: bool, } pub struct Context { @@ -57,6 +60,34 @@ impl ProxyHttp for LB { x4xx_limit: None, } } + + fn cache_key_callback(&self, session: &Session, _ctx: &mut Self::CTX) -> Result { + Ok(CacheKey::new(_ctx.hostname.clone().unwrap_or_default().as_ref(), session.req_header().uri.to_string(), "")) + } + fn request_cache_filter(&self, session: &mut Session, _ctx: &mut Self::CTX) -> Result<()> { + if self.cache_enabled && session.req_header().method == Method::GET { + if let Some(eviction) = EVICTION.get() { + session.cache.enable(&*MEM_CACHE, Some(eviction), None, Some(&*CACHE_LOCK), None); + } + } + Ok(()) + } + fn response_cache_filter(&self, _session: &Session, resp: &ResponseHeader, _ctx: &mut Self::CTX) -> Result { + let cc = CacheControl::from_resp_headers(resp); + let defaults = CacheMetaDefaults::new( + |status| { + if status.as_u16() == 200 { + Some(Duration::from_secs(*CACHE_TTL.get().unwrap_or(&60))) + } else { + None + } + }, + 0, + 0, + ); + Ok(pingora_cache::filters::resp_cacheable(cc.as_ref(), resp.clone(), false, &defaults)) + } + async fn request_filter(&self, session: &mut Session, _ctx: &mut Self::CTX) -> Result { ACTIVE_SESSIONS.inc(); let hostname = return_header_host_from_upstream(session, &self.ump_upst); @@ -159,7 +190,6 @@ impl ProxyHttp for LB { Some(hostname) => match ctx.upstream_peer.as_ref() { Some(innermap) => { let mut peer = Box::new(HttpPeer::new((&*innermap.address, innermap.port), innermap.is_ssl, hostname.to_string())); - if innermap.is_http2 { peer.options.alpn = ALPN::H2; } @@ -308,6 +338,13 @@ impl ProxyHttp for LB { } } } + // match session.cache.phase() { + // CachePhase::Hit => info!("Cache Status: HIT (Served from Memory)"), + // CachePhase::Miss => info!("Cache Status: MISS (Fetched from Upstream)"), + // CachePhase::Expired => info!("Cache Status: EXPIRED (Revalidating)"), + // _ => {} + // } + access_log(response_code, &self.request_summary(session, ctx), session); } } diff --git a/src/web/start.rs b/src/web/start.rs index ebf5f9c..2728a67 100644 --- a/src/web/start.rs +++ b/src/web/start.rs @@ -2,6 +2,7 @@ use crate::tls::grades; use crate::tls::load; use crate::tls::load::CertificateConfig; +use crate::utils::lazylock::{CACHE_TTL, EVICTION}; use crate::utils::structs::Extraparams; use crate::utils::tools::*; use crate::web::logging::init_access_log; @@ -10,6 +11,7 @@ use arc_swap::ArcSwap; use dashmap::DashMap; use log::info; use pingora::tls::ssl::{SslAlert, SslRef}; +use pingora_cache::eviction::simple_lru::Manager; use pingora_core::listeners::tls::TlsSettings; use pingora_core::listeners::TcpSocketOptions; use pingora_core::prelude::{background_service, Opt}; @@ -32,6 +34,13 @@ pub fn run() { let file = parameters.conf.clone().unwrap(); let maincfg = crate::utils::parceyaml::parce_main_config(file.as_str()); + let cache_enabled = maincfg.cache_size_mb.is_some(); + CACHE_TTL.set(maincfg.cache_ttl.unwrap_or(60)).unwrap(); + if let Some(size) = maincfg.cache_size_mb { + let cache_size = size * 1024 * 1024; + drop(EVICTION.set(Manager::new(cache_size))); + } + let mut server = Server::new(parameters).unwrap(); server.bootstrap(); @@ -59,6 +68,7 @@ pub fn run() { client_headers: ch_config, server_headers: sh_config, extraparams: ec_config, + cache_enabled: cache_enabled, }; let al = cfg.access_log.clone().unwrap_or("none".to_string()); init_access_log(al.as_str());