From e29161965f2702ddca8b46b904e8c24a650639b0 Mon Sep 17 00:00:00 2001 From: Ara Sadoyan Date: Tue, 26 May 2026 19:34:10 +0200 Subject: [PATCH] Added more monitoring metrics --- Cargo.lock | 92 +++++++++++++++++++++++++++++--------------- Cargo.toml | 14 ++++--- src/main.rs | 5 ++- src/utils/metrics.rs | 24 +++++++++++- src/web/webserver.rs | 33 ++-------------- 5 files changed, 99 insertions(+), 69 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index c0c5414..082ec3c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -142,7 +142,6 @@ dependencies = [ "jsonwebtoken", "log", "log4rs", - "mimalloc", "moka", "notify", "pingora", @@ -161,6 +160,8 @@ dependencies = [ "serde_yml", "sha2 0.11.0", "subtle", + "tikv-jemalloc-ctl", + "tikv-jemallocator", "tokio", "tonic", "tower-http", @@ -359,6 +360,15 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1938,15 +1948,6 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" -[[package]] -name = "libmimalloc-sys" -version = "0.1.47" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d1eacfa31c33ec25e873c136ba5669f00f9866d0688bea7be4d3f7e43067df6" -dependencies = [ - "cc", -] - [[package]] name = "libyml" version = "0.0.5" @@ -1995,9 +1996,9 @@ dependencies = [ [[package]] name = "log" -version = "0.4.29" +version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" +checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" dependencies = [ "serde_core", ] @@ -2073,15 +2074,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "mimalloc" -version = "0.1.50" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3627c4272df786b9260cabaa46aec1d59c93ede723d4c3ef646c503816b0640" -dependencies = [ - "libmimalloc-sys", -] - [[package]] name = "mime" version = "0.3.17" @@ -2494,6 +2486,12 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + [[package]] name = "pem" version = "3.0.6" @@ -3147,9 +3145,9 @@ checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rcgen" -version = "0.14.7" +version = "0.14.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10b99e0098aa4082912d4c649628623db6aba77335e4f4569ff5083a6448b32e" +checksum = "57f6d249aad744e274e682777a50283a225a32705394ee6d5fcc01efa25e4055" dependencies = [ "aws-lc-rs", "pem", @@ -3200,9 +3198,9 @@ checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" [[package]] name = "reqwest" -version = "0.13.3" +version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62e0021ea2c22aed41653bc7e1419abb2c97e038ff2c33d0e1309e49a97deec0" +checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3" dependencies = [ "base64", "bytes", @@ -3550,9 +3548,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.149" +version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" +checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ "itoa", "memchr", @@ -3921,6 +3919,37 @@ dependencies = [ "trackable", ] +[[package]] +name = "tikv-jemalloc-ctl" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a184c43b8ab2f41df2733b55556e3f5f632f4aeaa205b1bb018f574b7f5f142" +dependencies = [ + "libc", + "paste", + "tikv-jemalloc-sys", +] + +[[package]] +name = "tikv-jemalloc-sys" +version = "0.7.1+5.3.1-0-g81034ce1f1373e37dc865038e1bc8eeecf559ce8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a2825c78386b4ae0314074867860ba9577875de945f05992c38815cbec327f0" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "tikv-jemallocator" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "249f09e49ab1609436f34c776e84231bead18d6a955f119f939bdc1d847561bd" +dependencies = [ + "libc", + "tikv-jemalloc-sys", +] + [[package]] name = "time" version = "0.3.47" @@ -4111,9 +4140,9 @@ dependencies = [ [[package]] name = "tower-http" -version = "0.6.10" +version = "0.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68d6fdd9f81c2819c9a8b0e0cd91660e7746a8e6ea2ba7c6b2b057985f6bcb51" +checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags 2.11.1", "bytes", @@ -4891,10 +4920,11 @@ checksum = "fdd20c5420375476fbd4394763288da7eb0cc0b8c11deed431a91562af7335d3" [[package]] name = "yasna" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e17bb3549cc1321ae1296b9cdc2698e2b6cb1992adfa19a8c72e5b7a738f44cd" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" dependencies = [ + "bit-vec", "time", ] diff --git a/Cargo.toml b/Cargo.toml index 6e4f6b1..65a81e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,11 +20,11 @@ pingora-proxy = "0.8.0" pingora-http = "0.8.0" pingora-limits = "0.8.0" async-trait = "0.1.89" -log = "0.4.29" +log = "0.4.30" futures = "0.3.32" notify = "9.0.0-rc.4" axum = { version = "0.8.9" } -reqwest = { version = "0.13.3", features = ["json", "stream", "blocking"] } +reqwest = { version = "0.13.4", features = ["json", "stream", "blocking"] } serde_yml = "0.0.12" rand = "0.10.1" base64 = "0.22.1" @@ -34,17 +34,19 @@ sha2 = { version = "0.11.0-rc.5", default-features = false } base16ct = { version = "1.0.0", features = ["alloc"] } urlencoding = "2.1.3" arc-swap = "1.9.1" -mimalloc = { version = "0.1.50", default-features = false } prometheus = "0.14.0" x509-parser = "0.18.1" rustls-pemfile = "2.2.0" -tower-http = { version = "0.6.10", features = ["fs"] } +tower-http = { version = "0.6.11", features = ["fs"] } privdrop = "0.5.6" ctrlc = "3.5.2" -serde_json = "1.0.149" +serde_json = "1.0.150" subtle = "2.6.1" moka = { version = "0.12.15", features = ["sync"] } ahash = "0.8.12" instant-acme = "0.8.5" -rcgen = "0.14.7" +rcgen = "0.14.8" log4rs = "1.4.0" +#mimalloc = { version = "0.1.52", default-features = false } +tikv-jemallocator = "0.7.0" +tikv-jemalloc-ctl = { version = "0.7.0", features = ["stats"] } diff --git a/src/main.rs b/src/main.rs index 043a15e..2a35e0e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,9 +1,12 @@ +use tikv_jemallocator::Jemalloc; + mod tls; mod utils; mod web; #[global_allocator] -static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; +static ALLOC: Jemalloc = Jemalloc; +// static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc; // pub static A: CountingAllocator = CountingAllocator; fn main() { diff --git a/src/utils/metrics.rs b/src/utils/metrics.rs index 15db04d..5378989 100644 --- a/src/utils/metrics.rs +++ b/src/utils/metrics.rs @@ -5,6 +5,7 @@ use prometheus::{register_histogram, register_int_counter, register_int_counter_ use std::sync::Arc; use std::sync::LazyLock; use std::time::Duration; +use tikv_jemalloc_ctl::{epoch, stats}; pub struct MetricTypes { pub method: Method, @@ -14,18 +15,28 @@ pub struct MetricTypes { pub version: Version, } +pub static OPEN_FILES: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_open_files", "Number of open file descriptors").unwrap()); +pub static MEMORY_USAGE: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_memory_bytes", "Total memory allocated in bytes").unwrap()); pub static ACTIVE_SESSIONS: LazyLock = LazyLock::new(|| register_int_gauge!("aralez_active_sessions", "Current number of active sessions").unwrap()); - pub static REQUEST_COUNT: LazyLock = LazyLock::new(|| register_int_counter!("aralez_requests_total", "Total number of requests handled by Aralez").unwrap()); pub static RESPONSE_CODES: LazyLock = LazyLock::new(|| register_int_counter_vec!("aralez_responses_total", "Responses grouped by status code", &["status"]).unwrap()); +// pub static RESPONSE_LATENCY: LazyLock = LazyLock::new(|| { +// register_histogram!( +// "aralez_response_latency_seconds", +// "Response latency in seconds", +// vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0] +// ) +// .unwrap() +// }); + pub static RESPONSE_LATENCY: LazyLock = LazyLock::new(|| { register_histogram!( "aralez_response_latency_seconds", "Response latency in seconds", - vec![0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0] + vec![0.001, 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 1.0, 2.0, 5.0] ) .unwrap() }); @@ -54,3 +65,12 @@ pub fn calc_metrics(metric_types: &MetricTypes) { REQUESTS_BY_UPSTREAM.with_label_values(&[metric_types.upstream.as_ref()]).inc(); RESPONSE_LATENCY.observe(metric_types.latency.as_secs_f64()); } + +pub fn get_memory_usage() -> usize { + epoch::mib().unwrap().advance().unwrap(); // refresh stats + stats::allocated::mib().unwrap().read().unwrap() // bytes allocated +} + +pub fn get_open_files() -> usize { + std::fs::read_dir("/proc/self/fd").map(|dir| dir.count()).unwrap_or(0) +} diff --git a/src/web/webserver.rs b/src/web/webserver.rs index 75e747d..047fb73 100644 --- a/src/web/webserver.rs +++ b/src/web/webserver.rs @@ -1,9 +1,8 @@ -// use std::net::SocketAddr; use crate::tls::acme::order::CHALLENGES; -// use axum_server::tls_openssl::OpenSSLConfig; use crate::tls::acme::{account, order}; use crate::utils::discovery::APIUpstreamProvider; use crate::utils::jwt::Claims; +use crate::utils::metrics::{get_memory_usage, get_open_files, MEMORY_USAGE, OPEN_FILES}; use crate::utils::structs::{Config, Configuration, UpstreamsDashMap}; use crate::utils::tools::{upstreams_liveness_json, upstreams_to_json}; use axum::body::Body; @@ -56,11 +55,6 @@ pub async fn run_server(config: &APIUpstreamProvider, mut to_return: Sender, Json(payload): Json) -> } async fn metrics() -> impl IntoResponse { + MEMORY_USAGE.set(get_memory_usage() as i64); + OPEN_FILES.set(get_open_files() as i64); + let metric_families = gather(); let encoder = TextEncoder::new(); let mut buffer = Vec::new(); @@ -281,13 +265,4 @@ pub async fn http01_challenge(axum::extract::Path(token): axum::extract::Path, masterkey: &str) -> bool { -// if let Some(s) = headers.get("x-api-key").and_then(|v| v.to_str().ok()).or(params.get("key").map(|s| s.as_str())) { -// if s.as_bytes().ct_eq(masterkey.as_bytes()).into() { -// return true; -// } -// } -// false -// } - // -- ⚝ by Dave -- in NeoVim ⚝ --