Code restructurization

src/core/bgservice.rs

@@ -0,0 +1,144 @@
+use crate::core::logging::init_logging;
+use crate::core::proxyhttp::LB;
+use crate::ingress::{APIUpstreamProvider, ConsulProvider, Discovery, FromFileProvider, KubernetesProvider};
+use crate::tls::acme::order::refresh_order;
+use crate::utils::metrics::calc_cache_metrics;
+use crate::utils::parceyaml::load_configuration;
+use crate::utils::tools::*;
+use crate::utils::types::Configuration;
+use crate::utils::*;
+use async_trait::async_trait;
+use dashmap::DashMap;
+use log::{error, info};
+use pingora_core::server::ShutdownWatch;
+use pingora_core::services::background::BackgroundService;
+use std::sync::Arc;
+use std::time::{SystemTime, UNIX_EPOCH};
+use tokio::sync::mpsc;
+
+#[async_trait]
+impl BackgroundService for LB {
+    async fn start(&self, mut shutdown: ShutdownWatch) {
+        info!("Starting background service"); // tx: Sender<Configuration>
+        let (tx, mut rx) = mpsc::channel::<Configuration>(1);
+        let tx_api = tx.clone();
+        let config = load_configuration(self.config.upstreams_conf.clone().as_str(), "filepath")
+            .await
+            .0
+            .expect("Failed to load configuration");
+
+        match config.typecfg.as_str() {
+            "file" => {
+                info!("Running File discovery, requested type is: {}", config.typecfg);
+                tx.send(config).await.unwrap();
+                let file_load = FromFileProvider {
+                    path: self.config.upstreams_conf.clone(),
+                };
+                // let _ = tokio::spawn(async move { file_load.start(tx).await });
+                drop(tokio::spawn(async move { file_load.start(tx).await }));
+            }
+            "kubernetes" => {
+                info!("Running Kubernetes discovery, requested type is: {}", config.typecfg);
+                let cf = Arc::from(config);
+                let kuber_load = KubernetesProvider { config: cf.clone() };
+                drop(tokio::spawn(async move { kuber_load.start(tx).await }));
+            }
+            "consul" => {
+                info!("Running Consul discovery, requested type is: {}", config.typecfg);
+                let cf = Arc::from(config);
+                let consul_load = ConsulProvider { config: cf.clone() };
+                drop(tokio::spawn(async move { consul_load.start(tx).await }));
+            }
+            _ => {
+                error!("Unknown discovery type: {}", config.typecfg);
+            }
+        }
+
+        let confdir = self.config.proxy_configs.clone().unwrap_or_else(|| "/tmp".to_string()) + "/autoconfigs";
+        let certdir = self.config.proxy_configs.clone().unwrap_or_else(|| "/tmp".to_string()) + "/certificates";
+
+        let api_load = APIUpstreamProvider {
+            address: self.config.config_address.clone(),
+            masterkey: self.config.master_key.clone(),
+            config_api_enabled: self.config.config_api_enabled,
+            upstreams_file: self.config.upstreams_conf.clone(),
+            config_dir: confdir.clone(),
+            certs_dir: certdir.clone(),
+            file_server_address: self.config.file_server_address.clone(),
+            file_server_folder: self.config.file_server_folder.clone(),
+            current_upstreams: self.ump_upst.clone(),
+            full_upstreams: self.ump_full.clone(),
+        };
+        drop(tokio::spawn(async move { api_load.start(tx_api).await }));
+
+        let uu = self.ump_upst.clone();
+        let ff = self.ump_full.clone();
+        let im = self.ump_byid.clone();
+        let (hc_method, hc_interval) = (self.config.hc_method.clone(), self.config.hc_interval);
+        drop(tokio::spawn(async move {
+            healthcheck::hc2(uu, ff, im, (&*hc_method.to_string(), hc_interval.to_string().parse().unwrap())).await
+        }));
+        drop(tokio::spawn(async move { calc_cache_metrics().await }));
+        drop(tokio::spawn(async move { refresh_order(certdir, confdir).await }));
+        init_logging(self.config.access_log.clone());
+        loop {
+            tokio::select! {
+                _ = shutdown.changed() => {
+                    break;
+                }
+                val = rx.recv() => {
+                    let start = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+                    if let Some(ss) = val {
+                        clone_dashmap_into(&ss.upstreams, &self.ump_full);
+                        clone_dashmap_into(&ss.upstreams, &self.ump_upst);
+                        clone_idmap_into(&ss.upstreams, &self.ump_byid);
+                        let current = self.extraparams.load_full();
+                        let mut new = (*current).clone();
+                        new.to_https = ss.extraparams.to_https;
+                        new.sticky_sessions = ss.extraparams.sticky_sessions;
+                        new.authentication = ss.extraparams.authentication.clone();
+                        new.rate_limit = ss.extraparams.rate_limit;
+                        new.x4xx_limit = ss.extraparams.x4xx_limit;
+                        self.extraparams.store(Arc::new(new));
+                        self.client_headers.clear();
+                        self.server_headers.clear();
+                        for entry in ss.upstreams.iter() {
+                            let global_key = entry.key().clone();
+                            let client_global_values = DashMap::new();
+                            let server_global_values = DashMap::new();
+
+                            let mut client_target_entry = ss.client_headers.entry(global_key.clone()).or_insert_with(DashMap::new);
+                            client_target_entry.extend(client_global_values);
+                            let mut server_target_entry = ss.server_headers.entry(global_key).or_insert_with(DashMap::new);
+                            server_target_entry.extend(server_global_values);
+                            self.server_headers.insert(server_target_entry.key().to_owned(), server_target_entry.value().to_owned());
+                        }
+                        for path in ss.client_headers.iter() {
+                            let path_key = path.key().clone();
+                            let path_headers = path.value().clone();
+                            self.client_headers.insert(path_key.clone(), path_headers);
+                            if let Some(global_headers) = ss.client_headers.get("GLOBAL_CLIENT_HEADERS") {
+                                if let Some(existing_headers) = self.client_headers.get_mut(&path_key) {
+                                    merge_headers(&existing_headers, &global_headers);
+                                }
+                            }
+                        }
+                        for path in ss.server_headers.iter() {
+                            let path_key = path.key().clone();
+                            let path_headers = path.value().clone();
+                            self.server_headers.insert(path_key.clone(), path_headers);
+                            if let Some(global_headers) = ss.server_headers.get("GLOBAL_SERVER_HEADERS") {
+                                if let Some(existing_headers) = self.server_headers.get_mut(&path_key) {
+                                    merge_headers(&existing_headers, &global_headers);
+                                }
+                            }
+                        }
+                    }
+                    let end = SystemTime::now().duration_since(UNIX_EPOCH).unwrap();
+                    let startup = end - start;
+                    info!("Loading upstreams took: {}ms", startup.as_millis());
+                }
+            }
+        }
+    }
+}

src/core/gethosts.rs

@@ -0,0 +1,124 @@
+use crate::core::proxyhttp::LB;
+use crate::utils::types::InnerMap;
+use std::sync::atomic::{AtomicUsize, Ordering};
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub struct GetHostsReturHeaders {
+    pub client_headers: Option<Vec<(String, Arc<str>)>>,
+    pub server_headers: Option<Vec<(String, Arc<str>)>>,
+}
+
+pub trait GetHost {
+    fn find_sticky_backend(&self, servers: &[Arc<InnerMap>], backend_id: Option<&str>) -> Option<Arc<InnerMap>>;
+    fn pick_backend(&self, servers: &[Arc<InnerMap>], index: &AtomicUsize, backend_id: Option<&str>) -> Option<Arc<InnerMap>>;
+    fn get_host(&self, peer: &str, path: &str, backend_id: Option<&str>) -> Option<Arc<InnerMap>>;
+    fn get_header(&self, peer: &str, path: &str) -> Option<GetHostsReturHeaders>;
+}
+impl GetHost for LB {
+    fn find_sticky_backend(&self, servers: &[Arc<InnerMap>], backend_id: Option<&str>) -> Option<Arc<InnerMap>> {
+        let b = backend_id?;
+        let bb = self.ump_byid.get(b)?;
+        let target = bb.value();
+        servers.iter().any(|s| s.address == target.address && s.port == target.port).then(|| target.clone())
+    }
+    fn pick_backend(&self, servers: &[Arc<InnerMap>], index: &AtomicUsize, backend_id: Option<&str>) -> Option<Arc<InnerMap>> {
+        if servers.is_empty() {
+            return None;
+        }
+        if let Some(target) = self.find_sticky_backend(servers, backend_id) {
+            return Some(target);
+        }
+        let idx = index.fetch_add(1, Ordering::Relaxed) % servers.len();
+        Some(servers[idx].clone())
+    }
+    fn get_host(&self, peer: &str, path: &str, backend_id: Option<&str>) -> Option<Arc<InnerMap>> {
+        let host_entry = self.ump_upst.get(peer)?;
+        let mut end = path.len();
+        loop {
+            let slice = &path[..end];
+            if let Some(entry) = host_entry.get(slice) {
+                let (servers, index) = entry.value();
+                if let Some(backend) = self.pick_backend(servers, index, backend_id) {
+                    return Some(backend);
+                }
+            }
+            if let Some(pos) = slice.rfind('/') {
+                end = pos;
+            } else {
+                break;
+            }
+        }
+        if let Some(entry) = host_entry.get("/") {
+            let (servers, index) = entry.value();
+            if let Some(backend) = self.pick_backend(servers, index, backend_id) {
+                return Some(backend);
+            }
+        }
+        None
+    }
+
+    fn get_header(&self, peer: &str, path: &str) -> Option<GetHostsReturHeaders> {
+        let client_entry = self.client_headers.get(peer);
+        let server_entry = self.server_headers.get(peer);
+        if client_entry.is_none() && server_entry.is_none() {
+            return None;
+        }
+        let mut current_path = path;
+        let mut clnt_match = None;
+        if let Some(client_entry) = client_entry {
+            loop {
+                if let Some(entry) = client_entry.get(current_path) {
+                    if !entry.value().is_empty() {
+                        clnt_match = Some(entry.value().clone());
+                        break;
+                    }
+                }
+                if current_path == "/" {
+                    break;
+                }
+                if let Some(pos) = current_path.rfind('/') {
+                    current_path = if pos == 0 { "/" } else { &current_path[..pos] };
+                } else {
+                    break;
+                }
+            }
+        }
+        current_path = path;
+        let mut serv_match = None;
+        if let Some(server_entry) = server_entry {
+            loop {
+                if let Some(entry) = server_entry.get(current_path) {
+                    if !entry.value().is_empty() {
+                        serv_match = Some(entry.value().clone());
+                        break;
+                    }
+                }
+                if current_path == "/" {
+                    if let Some(entry) = server_entry.get("/") {
+                        if !entry.value().is_empty() {
+                            serv_match = Some(entry.value().clone());
+                            break;
+                        }
+                    }
+                    break;
+                }
+                if let Some(pos) = current_path.rfind('/') {
+                    current_path = if pos == 0 { "/" } else { &current_path[..pos] };
+                } else {
+                    break;
+                }
+            }
+        }
+        let result = GetHostsReturHeaders {
+            client_headers: clnt_match,
+            server_headers: serv_match,
+        };
+
+        if result.client_headers.is_some() || result.server_headers.is_some() {
+            Some(result)
+        } else {
+            None
+        }
+    }
+}

src/core/logging.rs

@@ -0,0 +1,213 @@
+use crate::utils::metrics::LOGGING_ERRORS;
+use crate::utils::types::AppConfig;
+use log::{error, info, warn, LevelFilter};
+use log4rs::append::rolling_file::policy::compound::roll::fixed_window::FixedWindowRoller;
+use log4rs::append::rolling_file::policy::compound::trigger::size::SizeTrigger;
+use log4rs::append::rolling_file::policy::compound::CompoundPolicy;
+use log4rs::append::rolling_file::RollingFileAppender;
+use log4rs::config::Logger;
+use log4rs::{
+    append::console::ConsoleAppender,
+    config::{Appender, Config as Log4rsConfig, Root},
+    encode::pattern::PatternEncoder,
+};
+use pingora_cache::CachePhase;
+use pingora_http::Version;
+use pingora_proxy::Session;
+use std::net::{IpAddr, Ipv4Addr};
+use std::sync::OnceLock;
+use tokio::sync::mpsc;
+
+#[derive(Debug)]
+pub struct LogMessage {
+    pub response_code: u16,
+    pub summary: String,
+    pub client_ip: IpAddr,
+    pub version: Version,
+    pub user_agent: String,
+    pub cache_status: CachePhase,
+}
+static LOG_SENDER: OnceLock<mpsc::Sender<LogMessage>> = OnceLock::new();
+static ACCESS_LOG: OnceLock<LogLevel> = OnceLock::new();
+const LOG_BUFFER: usize = 16384;
+
+pub fn log_builder(conf: &AppConfig, location: &Option<String>) {
+    let log_level = match conf.log_level.as_str() {
+        "info" => LevelFilter::Info,
+        "error" => LevelFilter::Error,
+        "warn" => LevelFilter::Warn,
+        "debug" => LevelFilter::Debug,
+        "trace" => LevelFilter::Trace,
+        "off" => LevelFilter::Off,
+        _ => {
+            println!("Error reading log level, defaulting to: INFO");
+            LevelFilter::Info
+        }
+    };
+
+    let mut pat: String = "{d(%Y-%m-%d %H:%M:%S)} {l} - {m} {n}".to_string();
+    if let Some(ptrn) = conf.log_pattern.clone() {
+        pat = ptrn;
+    }
+    let pattern = pat.as_str();
+
+    if let Some(location) = location {
+        let parts: Vec<&str> = location.splitn(4, ',').map(|s| s.trim()).collect();
+
+        let path = parts.get(0).expect("Syntax error, could not get path for log files");
+        let compress = parts.get(3).unwrap_or(&"No");
+        let size_mb: u64 = parts.get(1).and_then(|s| s.parse().ok()).unwrap_or(100);
+        let keep: u32 = parts.get(2).and_then(|s| s.parse().ok()).unwrap_or(5);
+
+        let pattern_str = match compress {
+            &"compress" => format!("{}.{{}}.gz", path),
+            _ => format!("{}.{{}}", path),
+        };
+
+        let roller = FixedWindowRoller::builder().build(pattern_str.as_str(), keep).unwrap();
+        let trigger = SizeTrigger::new(size_mb * 1024 * 1024);
+        let policy = CompoundPolicy::new(Box::new(trigger), Box::new(roller));
+
+        let file = RollingFileAppender::builder()
+            .encoder(Box::new(PatternEncoder::new(pattern)))
+            .build(path, Box::new(policy))
+            .unwrap();
+
+        let config = Log4rsConfig::builder()
+            .appender(Appender::builder().build("file", Box::new(file)))
+            .logger(Logger::builder().build("pingora_proxy", LevelFilter::Off))
+            .build(Root::builder().appender("file").build(log_level))
+            .unwrap();
+        log4rs::init_config(config).unwrap();
+        info!("Logging to: {}, Max file size: {}mb, Files to keep: {}, compression: {} ", path, size_mb, keep, compress);
+    } else {
+        let stdout = ConsoleAppender::builder().encoder(Box::new(PatternEncoder::new(pattern))).build();
+
+        let config = Log4rsConfig::builder()
+            .appender(Appender::builder().build("stdout", Box::new(stdout)))
+            .logger(Logger::builder().build("pingora_proxy", LevelFilter::Off))
+            .build(Root::builder().appender("stdout").build(log_level))
+            .unwrap();
+        log4rs::init_config(config).unwrap();
+        info!("No files are configured, logging to stdout");
+    }
+}
+
+pub fn init_access_log(level_str: &str) {
+    let level = LogLevel::from_str(level_str);
+    let _ = ACCESS_LOG.set(level);
+}
+
+#[derive(Debug)]
+pub enum LogLevel {
+    Access,
+    Error,
+    None,
+}
+
+impl LogLevel {
+    pub fn from_str(s: &str) -> Self {
+        match s {
+            "all" => LogLevel::Access,
+            "error" => LogLevel::Error,
+            _ => LogLevel::None,
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum MatchStatus {
+    Ok2xx,
+    Er4xx,
+    Er5xx,
+}
+impl MatchStatus {
+    #[inline]
+    pub fn from_code(code: u16) -> Self {
+        match code {
+            100..=399 => Self::Ok2xx,
+            400..=499 => Self::Er4xx,
+            _ => Self::Er5xx,
+        }
+    }
+}
+pub fn access_log(response_code: u16, summary: &str, session: &Session) {
+    let level = ACCESS_LOG.get().unwrap_or(&LogLevel::None);
+    let status = MatchStatus::from_code(response_code);
+    let should_log = match level {
+        LogLevel::Access => true,
+        LogLevel::None => false,
+        LogLevel::Error => matches!(status, MatchStatus::Er5xx),
+    };
+
+    if !should_log {
+        return;
+    }
+
+    let ip = session
+        .client_addr()
+        .and_then(|addr| addr.as_inet())
+        .map(|addr| addr.ip())
+        .unwrap_or(IpAddr::V4(Ipv4Addr::LOCALHOST));
+
+    let user_agent = session.req_header().headers.get("user-agent").and_then(|v| v.to_str().ok()).unwrap_or("-");
+    let log = LogMessage {
+        response_code,
+        summary: summary.to_owned(),
+        client_ip: ip,
+        version: session.req_header().version,
+        user_agent: user_agent.to_owned(),
+        cache_status: session.cache.phase(),
+    };
+
+    if let Some(sender) = LOG_SENDER.get() {
+        let sender = sender;
+        if let Err(_) = sender.try_send(log) {
+            LOGGING_ERRORS.inc();
+        }
+    }
+}
+
+pub fn init_logging(enabled: Option<String>) {
+    if let Some(_) = enabled {
+        LOGGING_ERRORS.set(0);
+        info!("Enabling {:?} log, with buffer of {} messages", ACCESS_LOG.get().unwrap_or(&LogLevel::None), LOG_BUFFER);
+        let (ltx, lrx) = mpsc::channel(LOG_BUFFER);
+        LOG_SENDER.set(ltx).unwrap();
+        std::thread::spawn(move || log_receiver(lrx));
+    }
+}
+
+pub fn log_receiver(mut receiver: mpsc::Receiver<LogMessage>) {
+    while let Some(msg) = receiver.blocking_recv() {
+        match MatchStatus::from_code(msg.response_code) {
+            MatchStatus::Ok2xx => info!(
+                "{}, {}, {}, client: {}, version: {:?}, useragent: {}",
+                msg.response_code,
+                msg.cache_status.as_str(),
+                msg.summary,
+                msg.client_ip,
+                msg.version,
+                msg.user_agent,
+            ),
+            MatchStatus::Er4xx => warn!(
+                "{}, {}, {}, client: {}, version: {:?}, useragent: {}",
+                msg.response_code,
+                msg.cache_status.as_str(),
+                msg.summary,
+                msg.client_ip,
+                msg.version,
+                msg.user_agent,
+            ),
+            MatchStatus::Er5xx => error!(
+                "{}, {}, {}, client: {}, version: {:?}, useragent: {}",
+                msg.response_code,
+                msg.cache_status.as_str(),
+                msg.summary,
+                msg.client_ip,
+                msg.version,
+                msg.user_agent,
+            ),
+        }
+    }
+}

src/core/mod.rs

@@ -0,0 +1,6 @@
+pub mod bgservice;
+pub mod gethosts;
+pub mod logging;
+pub mod proxyhttp;
+pub mod start;
+pub mod webserver;

src/core/proxyhttp.rs

@@ -0,0 +1,371 @@
+use crate::auth::authenticate;
+use crate::core::gethosts::{GetHost, GetHostsReturHeaders};
+use crate::core::logging::access_log;
+use crate::utils::lazylock::{CACHE_LOCK, CACHE_TTL, EVICTION, LOCALHOST, MEM_CACHE, RATE_LIMITER, REQUESTS_4XX, REVERSE_STORE};
+use crate::utils::metrics::*;
+use crate::utils::types::{AppConfig, Extraparams, Headers, InnerMap, UpstreamsDashMap, UpstreamsIdMap};
+use arc_swap::ArcSwap;
+use async_trait::async_trait;
+use axum::body::Bytes;
+use log::error;
+use pingora::http::{Method, RequestHeader, ResponseHeader, StatusCode};
+use pingora::prelude::*;
+use pingora::ErrorSource::Upstream;
+use pingora_cache::cache_control::CacheControl;
+use pingora_cache::{CacheKey, CacheMetaDefaults, RespCacheable};
+use pingora_core::listeners::ALPN;
+use pingora_core::prelude::HttpPeer;
+use pingora_proxy::{ProxyHttp, Session};
+use sha2::{Digest, Sha256};
+use std::cell::RefCell;
+use std::fmt::Write;
+use std::sync::Arc;
+use std::time::Duration;
+use tokio::time::Instant;
+
+thread_local! {static IP_BUFFER: RefCell<String> = RefCell::new(String::with_capacity(50));}
+#[derive(Clone)]
+pub struct LB {
+    pub ump_upst: Arc<UpstreamsDashMap>,
+    pub ump_full: Arc<UpstreamsDashMap>,
+    pub ump_byid: Arc<UpstreamsIdMap>,
+    pub client_headers: Arc<Headers>,
+    pub server_headers: Arc<Headers>,
+    pub config: Arc<AppConfig>,
+    pub extraparams: Arc<ArcSwap<Extraparams>>,
+    pub cache_enabled: bool,
+}
+
+pub struct Context {
+    backend_id: Option<String>,
+    start_time: Instant,
+    hostname: Option<Arc<str>>,
+    upstream_peer: Option<Arc<InnerMap>>,
+    extraparams: arc_swap::Guard<Arc<Extraparams>>,
+    client_headers: Option<Vec<(String, Arc<str>)>>,
+    x4xx_limit: Option<u32>,
+}
+
+thread_local! {
+    static KEY_BUF: RefCell<String> = RefCell::new(String::with_capacity(256));
+}
+#[async_trait]
+impl ProxyHttp for LB {
+    type CTX = Context;
+    fn new_ctx(&self) -> Self::CTX {
+        Context {
+            backend_id: None,
+            start_time: Instant::now(),
+            hostname: None,
+            upstream_peer: None,
+            extraparams: self.extraparams.load(),
+            client_headers: None,
+            x4xx_limit: None,
+        }
+    }
+
+    fn cache_key_callback(&self, session: &Session, ctx: &mut Self::CTX) -> Result<CacheKey> {
+        let host = ctx.hostname.as_deref().unwrap_or("default");
+        let path = session.req_header().uri.path();
+
+        let primary_key = KEY_BUF.with(|buf| {
+            let mut b = buf.borrow_mut();
+            b.clear();
+            b.push_str(host);
+            b.push_str(path);
+            b.clone()
+        });
+
+        let secondary_key = session.req_header().uri.path_and_query().map(|pq| pq.as_str()).unwrap_or("/");
+        Ok(CacheKey::new(primary_key, secondary_key))
+    }
+
+    fn request_cache_filter(&self, session: &mut Session, _ctx: &mut Self::CTX) -> Result<()> {
+        if self.cache_enabled && session.req_header().method == Method::GET {
+            if let Some(eviction) = EVICTION.get() {
+                session.cache.enable(&*MEM_CACHE, Some(eviction), None, Some(&*CACHE_LOCK), None);
+            }
+        }
+        Ok(())
+    }
+    fn response_cache_filter(&self, _session: &Session, resp: &ResponseHeader, _ctx: &mut Self::CTX) -> Result<RespCacheable> {
+        let cc = CacheControl::from_resp_headers(resp);
+        let defaults = CacheMetaDefaults::new(
+            |status| {
+                if status == StatusCode::OK {
+                    Some(Duration::from_secs(*CACHE_TTL.get().unwrap_or(&60)))
+                } else {
+                    None
+                }
+            },
+            0,
+            0,
+        );
+        Ok(pingora_cache::filters::resp_cacheable(cc.as_ref(), resp.clone(), false, &defaults))
+    }
+
+    async fn request_filter(&self, session: &mut Session, _ctx: &mut Self::CTX) -> Result<bool> {
+        ACTIVE_SESSIONS.inc();
+        let hostname = return_header_host_from_upstream(session, &self.ump_upst);
+        _ctx.hostname = hostname;
+        let mut backend_id = None;
+        if let Some(_) = _ctx.extraparams.sticky_sessions {
+            if let Some(cookies) = session.req_header().headers.get("cookie") {
+                if let Ok(cookie_str) = cookies.to_str() {
+                    if let Some(pos) = cookie_str.find("backend_id=") {
+                        let value = &cookie_str[pos + "backend_id=".len()..];
+                        let end = value.find(';').unwrap_or(value.len());
+                        backend_id = Some(&value[..end]);
+                    }
+                }
+            }
+        }
+        match _ctx.hostname.as_ref() {
+            None => return Ok(false),
+            Some(host) => {
+                let optioninnermap = self.get_host(host, session.req_header().uri.path(), backend_id);
+                match optioninnermap {
+                    None => return Ok(false),
+                    Some(ref innermap) => {
+                        if let Some(auth) = _ctx.extraparams.authentication.as_ref().or(innermap.authorization.as_ref()) {
+                            if !authenticate(&auth, session).await {
+                                let _ = session.respond_error(401).await;
+                                return Ok(true);
+                            }
+                        }
+                        if let Some(rate) = innermap.x4xx_limit.or(_ctx.extraparams.x4xx_limit) {
+                            _ctx.x4xx_limit = innermap.x4xx_limit;
+                            let rate_key = session.client_addr().and_then(|addr| addr.as_inet()).map(|inet| inet.ip());
+                            if let Some(rk) = rate_key {
+                                let count = REQUESTS_4XX.get(&rk).unwrap_or(0);
+                                if count > rate {
+                                    let header = ResponseHeader::build(429, None)?;
+                                    session.set_keepalive(None);
+                                    session.write_response_header(Box::new(header), true).await?;
+                                    return Ok(true);
+                                }
+                            }
+                        }
+                        if let Some(rate) = innermap.rate_limit.or(_ctx.extraparams.rate_limit) {
+                            let rate_key = session.client_addr().and_then(|addr| addr.as_inet()).map(|inet| inet.ip());
+                            let curr_window_requests = RATE_LIMITER.observe(&rate_key, 1);
+                            if curr_window_requests > rate {
+                                let header = ResponseHeader::build(429, None)?;
+                                session.set_keepalive(None);
+                                session.write_response_header(Box::new(header), true).await?;
+                                return Ok(true);
+                            }
+                        }
+
+                        if let Some(redirect_to) = &innermap.redirect_to {
+                            let uri = session.req_header().uri.path();
+                            let capacity = redirect_to.len() + uri.len();
+                            let mut s = String::with_capacity(capacity);
+                            s.push_str(redirect_to);
+                            s.push_str(uri);
+                            let mut resp = ResponseHeader::build(StatusCode::MOVED_PERMANENTLY, None)?;
+                            resp.insert_header("Location", s)?;
+                            resp.insert_header("Content-Length", "0")?;
+                            session.write_response_header(Box::new(resp), true).await?;
+                            return Ok(true);
+                        }
+
+                        if _ctx.extraparams.to_https.unwrap_or(false) || innermap.to_https {
+                            if let Some(stream) = session.stream() {
+                                if stream.get_ssl().is_none() {
+                                    if let Some(host) = _ctx.hostname.as_ref() {
+                                        let port = self.config.proxy_port_tls.as_deref().unwrap_or("443");
+                                        let uri = session.req_header().uri.path();
+                                        let capacity = host.len() + uri.len() + 8;
+                                        let mut s = String::with_capacity(capacity);
+                                        s.push_str("https://");
+                                        s.push_str(host);
+                                        if port != "443" {
+                                            s.push(':');
+                                            s.push_str(port);
+                                        }
+                                        s.push_str(uri);
+                                        let mut resp = ResponseHeader::build(StatusCode::MOVED_PERMANENTLY, None)?;
+                                        resp.insert_header("Location", s)?;
+                                        resp.insert_header("Content-Length", "0")?;
+                                        session.write_response_header(Box::new(resp), true).await?;
+                                        return Ok(true);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                }
+                _ctx.upstream_peer = optioninnermap;
+            }
+        }
+        Ok(false)
+    }
+    async fn upstream_peer(&self, session: &mut Session, ctx: &mut Self::CTX) -> Result<Box<HttpPeer>> {
+        match ctx.hostname.as_ref() {
+            Some(hostname) => match ctx.upstream_peer.as_ref() {
+                Some(innermap) => {
+                    let mut peer = Box::new(HttpPeer::new((&*innermap.address, innermap.port), innermap.is_ssl, hostname.to_string()));
+                    if innermap.is_http2 && !session.is_upgrade_req() {
+                        peer.options.alpn = ALPN::H2;
+                    }
+                    if innermap.is_ssl {
+                        peer.options.verify_cert = false;
+                        peer.options.verify_hostname = false;
+                    }
+                    /*
+                    Experimental optionsv
+                    The following TCP optimizations were tested but caused performance degrade under heavy load:
+                    peer.options.tcp_keepalive = Some(TcpKeepalive {
+                        idle: Duration::from_secs(60),
+                        interval: Duration::from_secs(10),
+                        count: 5,
+                        user_timeout: Duration::from_secs(30),
+                    });
+
+                    peer.options.idle_timeout = Some(Duration::from_secs(300));
+                    peer.options.tcp_recv_buf = Some(128 * 1024);
+                    End of experimental options
+                    */
+                    if let Some(_) = ctx.extraparams.sticky_sessions {
+                        let mut s = String::with_capacity(64);
+                        write!(
+                            &mut s,
+                            "{}:{}:{}:{}:{}:{}:{}:{}:{:?}",
+                            hostname,
+                            innermap.address,
+                            innermap.port,
+                            innermap.is_http2,
+                            innermap.to_https,
+                            innermap.x4xx_limit.unwrap_or_default(),
+                            innermap.rate_limit.unwrap_or_default(),
+                            innermap.healthcheck.unwrap_or_default(),
+                            innermap.authorization
+                        )
+                        .unwrap_or(());
+                        ctx.backend_id = Some(s);
+                    }
+                    Ok(peer)
+                }
+                None => {
+                    if let Err(e) = session.respond_error_with_body(502, Bytes::from("502 Bad Gateway\n")).await {
+                        error!("Failed to send error response: {:?}", e);
+                    }
+                    Err(Box::new(Error {
+                        etype: HTTPStatus(502),
+                        esource: Upstream,
+                        retry: RetryType::Decided(false),
+                        cause: None,
+                        context: Option::from(ImmutStr::Static("Upstream not found")),
+                    }))
+                }
+            },
+            None => {
+                if let Err(e) = session.respond_error_with_body(502, Bytes::from("502 Bad Gateway\n")).await {
+                    error!("Failed to send error response: {:?}", e);
+                }
+                Err(Box::new(Error {
+                    etype: HTTPStatus(502),
+                    esource: Upstream,
+                    retry: RetryType::Decided(false),
+                    cause: None,
+                    context: None,
+                }))
+            }
+        }
+    }
+
+    async fn upstream_request_filter(&self, session: &mut Session, upstream_request: &mut RequestHeader, ctx: &mut Self::CTX) -> Result<()> {
+        if let Some(ip) = session.client_addr().and_then(|a| a.as_inet()).map(|i| i.ip()) {
+            IP_BUFFER.with(|buffer| {
+                let mut buf = buffer.borrow_mut();
+                buf.clear();
+                write!(buf, "{}", ip).unwrap_or(());
+                upstream_request.append_header("X-Forwarded-For", buf.as_str()).unwrap_or(false);
+            });
+        }
+
+        let hostname = ctx.hostname.as_deref().unwrap_or("localhost");
+        let path = session.req_header().uri.path();
+        let GetHostsReturHeaders { server_headers, client_headers } = match self.get_header(hostname, path) {
+            Some(h) => h,
+            None => return Ok(()),
+        };
+
+        if let Some(sh) = server_headers {
+            for (k, v) in sh {
+                upstream_request.insert_header(k, v.as_ref())?;
+            }
+        }
+        if let Some(ch) = client_headers {
+            ctx.client_headers = Some(ch);
+        }
+        Ok(())
+    }
+    async fn response_filter(&self, _session: &mut Session, _upstream_response: &mut ResponseHeader, ctx: &mut Self::CTX) -> Result<()> {
+        if let Some(val) = ctx.extraparams.sticky_sessions {
+            if let Some(bid) = &ctx.backend_id {
+                let tt = if let Some(existing) = REVERSE_STORE.get(bid) {
+                    existing.value().clone()
+                } else {
+                    let mut hasher = Sha256::new();
+                    hasher.update(bid.as_bytes());
+                    let hash = hasher.finalize();
+                    let hex_hash = base16ct::lower::encode_string(&hash);
+                    let hh = hex_hash[0..50].to_string();
+                    REVERSE_STORE.insert(bid.clone(), hh.clone());
+                    REVERSE_STORE.insert(hh.clone(), bid.clone());
+                    hh
+                };
+                let mut buf = String::with_capacity(80);
+                buf.push_str("backend_id=");
+                buf.push_str(&tt);
+                buf.push_str("; Path=/; Max-Age=");
+                buf.push_str(&val.to_string());
+                buf.push_str("; HttpOnly; SameSite=Lax");
+                let _ = _upstream_response.append_header("set-cookie", buf.as_str());
+            }
+        }
+        Ok(())
+    }
+    async fn upstream_response_filter(&self, _session: &mut Session, upstream_response: &mut ResponseHeader, ctx: &mut Self::CTX) -> Result<()> {
+        if let Some(client_headers) = &ctx.client_headers {
+            for (k, v) in client_headers.iter() {
+                upstream_response.append_header(k.clone(), v.as_ref())?;
+            }
+        }
+        Ok(())
+    }
+    async fn logging(&self, session: &mut Session, _e: Option<&pingora::Error>, ctx: &mut Self::CTX) {
+        let response_code = session.response_written().map_or(0, |resp| resp.status.as_u16());
+        let m = &MetricTypes {
+            method: session.req_header().method.clone(),
+            code: session.response_written().map(|resp| resp.status),
+            latency: ctx.start_time.elapsed(),
+            version: session.req_header().version,
+            upstream: ctx.hostname.take().unwrap_or_else(|| LOCALHOST.clone()),
+        };
+        calc_metrics(m);
+        ACTIVE_SESSIONS.dec();
+        if let Some(_) = ctx.x4xx_limit.or(ctx.extraparams.x4xx_limit) {
+            if (400..=499).contains(&response_code) {
+                if let Some(ip) = session.client_addr().and_then(|a| a.as_inet()).map(|i| i.ip()) {
+                    let current = REQUESTS_4XX.get(&ip).unwrap_or(0);
+                    REQUESTS_4XX.insert(ip, current + 1);
+                }
+            }
+        }
+        access_log(response_code, &self.request_summary(session, ctx), session);
+    }
+}
+
+fn return_header_host_from_upstream(session: &Session, ump_upst: &UpstreamsDashMap) -> Option<Arc<str>> {
+    let host_str = if session.is_http2() {
+        session.req_header().uri.host()?
+    } else {
+        let h = session.req_header().headers.get("host")?.to_str().ok()?;
+        h.split_once(':').map_or(h, |(host, _)| host)
+    };
+
+    ump_upst.get(host_str).or_else(|| ump_upst.get("DEFAULT")).map(|entry| entry.key().clone())
+}

src/core/start.rs

@@ -0,0 +1,181 @@
+use crate::core::logging::init_access_log;
+use crate::core::proxyhttp::LB;
+use crate::tls::grades;
+use crate::tls::load;
+use crate::tls::load::CertificateConfig;
+use crate::utils::folder_watch;
+use crate::utils::lazylock::{CACHE_TTL, EVICTION};
+use crate::utils::tools::*;
+use crate::utils::types::Extraparams;
+use arc_swap::ArcSwap;
+use dashmap::DashMap;
+use log::info;
+use pingora::tls::ssl::{SslAlert, SslRef};
+use pingora_cache::eviction::simple_lru::Manager;
+use pingora_core::listeners::tls::TlsSettings;
+use pingora_core::listeners::TcpSocketOptions;
+use pingora_core::prelude::{background_service, Opt};
+use pingora_core::protocols::TcpKeepalive;
+use pingora_core::server::Server;
+use privdrop::reexports::libc::SIGQUIT;
+use rustls::crypto::ring::default_provider;
+use sd_notify::NotifyState;
+use signal_hook::{
+    consts::{SIGINT, SIGTERM},
+    iterator::Signals,
+};
+use std::sync::mpsc::{channel, Receiver, Sender};
+use std::sync::Arc;
+use std::time::Duration;
+use std::{fs, thread};
+
+pub fn run() {
+    default_provider().install_default().expect("Failed to install RusTLS crypto provider");
+    let parameters = Opt::parse_args();
+    let file = parameters.conf.clone().unwrap();
+    let maincfg = crate::utils::parceyaml::parce_main_config(file.as_str());
+
+    let cache_enabled = maincfg.cache_size_mb.is_some();
+    CACHE_TTL.set(maincfg.cache_ttl.unwrap_or(60)).unwrap();
+    if let Some(size) = maincfg.cache_size_mb {
+        let cache_size = size * 1024 * 1024;
+        drop(EVICTION.set(Manager::new(cache_size)));
+    }
+
+    let mut server = Server::new(parameters).unwrap();
+    server.bootstrap();
+
+    let uf_config = Arc::new(DashMap::new());
+    let ff_config = Arc::new(DashMap::new());
+    let im_config = Arc::new(DashMap::new());
+    let ch_config = Arc::new(DashMap::new());
+    let sh_config = Arc::new(DashMap::new());
+
+    let ec_config = Arc::new(ArcSwap::from_pointee(Extraparams {
+        to_https: None,
+        sticky_sessions: None,
+        authentication: None,
+        rate_limit: None,
+        x4xx_limit: None,
+    }));
+
+    let cfg = Arc::new(maincfg);
+
+    let lb = LB {
+        ump_upst: uf_config,
+        ump_full: ff_config,
+        ump_byid: im_config,
+        config: cfg.clone(),
+        client_headers: ch_config,
+        server_headers: sh_config,
+        extraparams: ec_config,
+        cache_enabled: cache_enabled,
+    };
+    let al = cfg.access_log.clone().unwrap_or("none".to_string());
+    init_access_log(al.as_str());
+
+    let grade = cfg.proxy_tls_grade.clone().unwrap_or("medium".to_string());
+    info!("TLS grade set to: [ {} ]", grade);
+
+    let bg_srvc = background_service("bgsrvc", lb.clone());
+    let bind_address_http = cfg.proxy_address_http.clone();
+    let bind_address_tls = cfg.proxy_address_tls.clone();
+
+    let mut proxy = pingora_proxy::http_proxy_service(&server.configuration, lb.clone());
+
+    check_priv(bind_address_http.as_str());
+
+    // let mut tcp_options: Option<TcpSocketOptions> = Some(TcpSocketOptions::default());
+    // let mut tcp_options = TcpSocketOptions::default();
+
+    let mut tcp_options: Option<TcpSocketOptions> = None;
+    if let Some(idle) = cfg.tcp_keepalive_idle {
+        let mut to = TcpSocketOptions::default();
+        to.tcp_keepalive = Some(TcpKeepalive {
+            idle: Duration::from_secs(idle),
+            interval: Duration::from_secs(cfg.tcp_keepalive_interval.unwrap_or(10)),
+            user_timeout: Default::default(),
+            count: cfg.tcp_keepalive_count.unwrap_or(5usize),
+        });
+        tcp_options = Some(to);
+        info!(
+            "Applying kernel tcp_keepalive parameters: idle {}, interval {}, count {}",
+            idle,
+            cfg.tcp_keepalive_interval.unwrap_or(60),
+            cfg.tcp_keepalive_count.unwrap_or(5),
+        );
+    }
+
+    if let Some(bind_address_tls) = bind_address_tls {
+        check_priv(bind_address_tls.as_str());
+        let (tx, rx): (Sender<Vec<CertificateConfig>>, Receiver<Vec<CertificateConfig>>) = channel();
+        let certs_path = cfg.proxy_configs.clone().unwrap() + "/certificates";
+
+        if fs::metadata(certs_path.clone()).is_err() {
+            fs::create_dir_all(certs_path.clone()).unwrap();
+        }
+        thread::spawn(move || {
+            folder_watch(certs_path, tx).unwrap();
+        });
+        let certificate_configs = rx.recv().unwrap();
+        let first_set = load::Certificates::new(&certificate_configs, grade.as_str()).unwrap_or_else(|| panic!("Unable to load initial certificate info"));
+        let certificates = Arc::new(ArcSwap::from_pointee(first_set));
+        let certs_for_callback = certificates.clone();
+
+        let certs_for_watcher = certificates.clone();
+        let new_certs = load::Certificates::new(&certificate_configs, grade.as_str());
+        certs_for_watcher.store(Arc::new(new_certs.unwrap()));
+
+        let mut tls_settings =
+            TlsSettings::intermediate(&certs_for_callback.load().default_cert_path, &certs_for_callback.load().default_key_path).expect("unable to load or parse cert/key");
+
+        grades::set_tsl_grade(&mut tls_settings, grade.as_str());
+        tls_settings.set_servername_callback(move |ssl_ref: &mut SslRef, ssl_alert: &mut SslAlert| certs_for_callback.load().server_name_callback(ssl_ref, ssl_alert));
+        tls_settings.set_alpn_select_callback(grades::prefer_h2);
+
+        proxy.add_tls_with_settings(&bind_address_tls, tcp_options.clone(), tls_settings);
+
+        let certs_for_watcher = certificates.clone();
+        thread::spawn(move || {
+            while let Ok(new_configs) = rx.recv() {
+                let new_certs = load::Certificates::new(&new_configs, grade.as_str());
+                if let Some(new_certs) = new_certs {
+                    certs_for_watcher.store(Arc::new(new_certs));
+                };
+            }
+        });
+    }
+    info!("Running HTTP listener on :{}", bind_address_http);
+    if let Some(tc) = tcp_options {
+        proxy.add_tcp_with_settings(&bind_address_http, tc);
+    } else {
+        proxy.add_tcp(&bind_address_http)
+    }
+
+    server.add_service(proxy);
+    server.add_service(bg_srvc);
+    thread::spawn(move || server.run_forever());
+
+    if let (Some(user), Some(group)) = (cfg.rungroup.clone(), cfg.runuser.clone()) {
+        drop_priv(user, group, cfg.proxy_address_http.clone(), cfg.proxy_address_tls.clone());
+    }
+    let _ = sd_notify::notify(&[NotifyState::Ready]);
+
+    let pf = cfg.pid_file.clone().unwrap_or("/tmp/aralez.pid".to_string());
+    if let Err(e) = write_pid_file(pf.as_str()) {
+        panic!("Failed to write PID file: {} : {}", pf, e);
+    }
+    let mut signals = Signals::new(&[SIGINT, SIGTERM, SIGQUIT]).unwrap();
+    for sig in signals.forever() {
+        match sig {
+            SIGINT => info!("SIGINT received! Exiting..."),
+            SIGTERM => info!("SIGTERM received! Exiting..."),
+            SIGQUIT => {
+                thread::sleep(Duration::from_secs(300));
+                info!("SIGQUIT received! Exiting...")
+            }
+            _ => unreachable!(),
+        }
+        break;
+    }
+}

src/core/webserver.rs

@@ -0,0 +1,259 @@
+use crate::auth::jwt::Claims;
+use crate::ingress::APIUpstreamProvider;
+use crate::tls::acme::http01::{acme_create, acme_order, http01_challenge};
+use crate::utils::metrics::{get_memory_usage, get_open_files, MEMORY_USAGE, OPEN_FILES};
+use crate::utils::tools::{upstreams_liveness_json, upstreams_to_json};
+use crate::utils::types::{Config, Configuration, UpstreamsDashMap};
+use axum::body::Body;
+use axum::extract::{Query, State};
+use axum::http::{Response, StatusCode};
+use axum::response::IntoResponse;
+use axum::routing::{any, get, post};
+use axum::{Json, Router};
+use jsonwebtoken::{encode, EncodingKey, Header};
+use log::{debug, error, info, warn};
+use prometheus::{gather, Encoder, TextEncoder};
+use serde::Serialize;
+use signal_hook::{consts::SIGQUIT, iterator::Signals};
+use std::collections::HashMap;
+use std::net::SocketAddr;
+use std::str::FromStr;
+use std::sync::Arc;
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
+use tokio::net::TcpListener;
+use tokio::sync::mpsc;
+use tower_http::services::ServeDir;
+
+#[derive(Serialize, Debug)]
+struct OutToken {
+    token: String,
+}
+
+#[derive(Clone)]
+pub struct AppState {
+    master_key: Option<String>,
+    pub cert_creds: String,
+    pub certs_dir: String,
+    upstreams_file: String,
+    config_sender: mpsc::Sender<Configuration>,
+    config_api_enabled: bool,
+    current_upstreams: Arc<UpstreamsDashMap>,
+    full_upstreams: Arc<UpstreamsDashMap>,
+}
+
+#[allow(unused_mut)]
+pub async fn run_server(config: &APIUpstreamProvider, mut to_return: mpsc::Sender<Configuration>, upstreams_curr: Arc<UpstreamsDashMap>, upstreams_full: Arc<UpstreamsDashMap>) {
+    let credsfile = config.config_dir.clone() + "/acme_credentials.json";
+    let app_state = AppState {
+        master_key: config.masterkey.clone(),
+        cert_creds: credsfile,
+        certs_dir: config.certs_dir.clone(),
+        upstreams_file: config.upstreams_file.clone(),
+        config_sender: to_return.clone(),
+        config_api_enabled: config.config_api_enabled,
+        current_upstreams: upstreams_curr,
+        full_upstreams: upstreams_full,
+    };
+    let app = Router::new()
+        // .route("/{*wildcard}", get(senderror))
+        .route("/jwt", post(jwt_gen))
+        .route("/acme_create", any(acme_create))
+        .route("/acme_order/{*domain}", any(acme_order))
+        .route("/.well-known/acme-challenge/{*token}", any(http01_challenge))
+        .route("/conf", post(conf))
+        .route("/metrics", get(metrics))
+        .route("/status", get(status))
+        .with_state(app_state);
+
+    let mut static_handle: Option<tokio::task::JoinHandle<()>> = None;
+    if let (Some(address), Some(folder)) = (&config.file_server_address, &config.file_server_folder) {
+        let static_listen = port_is_available("File Server", &address).await;
+        let static_files = ServeDir::new(folder);
+        let static_serve: Router = Router::new().fallback_service(static_files);
+        info!("Starting file server on: {}", address);
+
+        static_handle = Some(tokio::spawn(async move {
+            match axum::serve(static_listen, static_serve).await {
+                Ok(o) => info!("File server exited normally: {:?}", o),
+                Err(e) => error!("File server exited: {}", e),
+            }
+        }))
+
+        // let a = config.file_server_address.to_owned();
+        // let f = config.file_server_folder.to_owned();
+        // drop(tokio::spawn(async move { run_file_server(a, f).await }));
+    }
+
+    let listener = port_is_available("Config API", &config.address).await;
+    info!("Starting the API server on: {}", config.address);
+    let api_server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+
+    let (tx, mut rx) = mpsc::channel(1);
+    std::thread::spawn(move || {
+        let mut signals = Signals::new(&[SIGQUIT]).unwrap();
+        for sig in signals.forever() {
+            tx.blocking_send(sig).unwrap();
+            break;
+        }
+    });
+    rx.recv().await;
+    api_server.abort();
+    if let Some(handle) = static_handle {
+        handle.abort();
+    }
+    info!("Exiting...");
+}
+
+async fn conf(State(st): State<AppState>, Query(params): Query<HashMap<String, String>>, content: String) -> impl IntoResponse {
+    if !st.config_api_enabled {
+        return Response::builder().status(StatusCode::FORBIDDEN).body(Body::from("Config API is disabled !\n")).unwrap();
+    }
+
+    let strcontent = content.as_str();
+    let parsed = noyalib::from_str::<Config>(strcontent);
+    match parsed {
+        Ok(_) => {
+            if let Some(_) = params.get("save") {
+                drop(tokio::spawn(async move { apply_config(content.as_str(), st, true).await }));
+            } else {
+                drop(tokio::spawn(async move { apply_config(content.as_str(), st, false).await }));
+            }
+            Response::builder().status(StatusCode::OK).body(Body::from("Accepted! Applying in background\n")).unwrap()
+        }
+        Err(err) => {
+            error!("Failed to parse upstreams file: {}", err);
+            Response::builder().status(StatusCode::BAD_GATEWAY).body(Body::from(format!("Failed: {}\n", err))).unwrap()
+        }
+    }
+}
+
+async fn apply_config(content: &str, st: AppState, save: bool) {
+    let sl = crate::utils::parceyaml::load_configuration(content, "content").await;
+    if let Some(serverlist) = sl.0 {
+        if save {
+            info!("Saving new upstreams to: {}", st.upstreams_file);
+            if let Err(err) = std::fs::write(&st.upstreams_file, content) {
+                error!("Error saving to: {} : {}", st.upstreams_file, err);
+            }
+        }
+        let _ = st.config_sender.send(serverlist).await;
+    }
+}
+
+async fn jwt_gen(State(state): State<AppState>, Json(payload): Json<Claims>) -> (StatusCode, Json<OutToken>) {
+    if let Some(master_key) = &state.master_key {
+        if &payload.master_key == master_key {
+            let now = SystemTime::now() + Duration::from_secs(payload.exp * 60);
+            let expire = now.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs();
+
+            let claim = Claims {
+                master_key: String::new(),
+                owner: payload.owner,
+                exp: expire,
+                random: payload.random,
+            };
+            match encode(&Header::default(), &claim, &EncodingKey::from_secret(payload.master_key.as_ref())) {
+                Ok(t) => {
+                    let tok = OutToken { token: t };
+                    debug!("Generating token: {:?}", tok.token);
+                    (StatusCode::CREATED, Json(tok))
+                }
+                Err(e) => {
+                    let tok = OutToken { token: "ERROR".to_string() };
+                    error!("Failed to generate token: {:?}", e);
+                    (StatusCode::INTERNAL_SERVER_ERROR, Json(tok))
+                }
+            }
+        } else {
+            let tok = OutToken {
+                token: "Unauthorised".to_string(),
+            };
+            warn!("Unauthorised JWT generate request: {:?}", tok);
+            (StatusCode::FORBIDDEN, Json(tok))
+        }
+    } else {
+        let tok = OutToken {
+            token: "ERROR Getting JWT_KEY environment variable".to_string(),
+        };
+        error!("ERROR Getting JWT_KEY environment variable");
+        (StatusCode::INTERNAL_SERVER_ERROR, Json(tok))
+    }
+}
+
+async fn metrics() -> impl IntoResponse {
+    MEMORY_USAGE.set(get_memory_usage() as i64);
+    OPEN_FILES.set(get_open_files() as i64);
+
+    let metric_families = gather();
+    let encoder = TextEncoder::new();
+    let mut buffer = Vec::new();
+    if let Err(e) = encoder.encode(&metric_families, &mut buffer) {
+        // encoding error fallback
+        return Response::builder()
+            .status(StatusCode::INTERNAL_SERVER_ERROR)
+            .body(Body::from(format!("Failed to encode metrics: {}", e)))
+            .unwrap();
+    }
+    Response::builder()
+        .status(StatusCode::OK)
+        .header("Content-Type", encoder.format_type())
+        .body(Body::from(buffer))
+        .unwrap()
+}
+
+#[allow(clippy::needless_return)]
+async fn status(State(st): State<AppState>, Query(params): Query<HashMap<String, String>>) -> impl IntoResponse {
+    if params.contains_key("live") {
+        let r = upstreams_liveness_json(&st.full_upstreams, &st.current_upstreams);
+        return Response::builder()
+            .status(StatusCode::OK)
+            .header("Content-Type", "application/json")
+            .body(Body::from(format!("{}", r)))
+            .unwrap();
+    }
+    if params.contains_key("all") {
+        let resp = upstreams_to_json(&st.current_upstreams);
+        match resp {
+            Ok(j) => {
+                return Response::builder()
+                    .status(StatusCode::OK)
+                    .header("Content-Type", "application/json")
+                    .body(Body::from(j))
+                    .unwrap()
+            }
+            Err(e) => {
+                return Response::builder()
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)
+                    .body(Body::from(format!("Failed to get status: {}", e)))
+                    .unwrap();
+            }
+        }
+    }
+    Response::builder()
+        .status(StatusCode::INTERNAL_SERVER_ERROR)
+        .body(Body::from("Parameter mismatch"))
+        .unwrap()
+}
+
+pub async fn port_is_available(name: &str, address: &str) -> TcpListener {
+    let addr = SocketAddr::from_str(address).unwrap_or_else(|e| panic!("{}: Invalid address format: {:?}", name, e));
+    let t = Duration::from_secs(2);
+
+    //if addr.ip() == IpAddr::V4(Ipv4Addr::UNSPECIFIED) {
+    //    addr.set_ip(IpAddr::V4(Ipv4Addr::LOCALHOST));
+    //}
+    let p = addr.port();
+    loop {
+        match TcpListener::bind(addr).await {
+            Ok(listener) => {
+                return listener;
+            }
+            Err(_) => {
+                warn!("{} port is not available: {} will try again in {:?}", name, p, t);
+                tokio::time::sleep(t).await;
+            }
+        }
+    }
+}
+
+// -- ⚝ by Dave -- in NeoVim ⚝ --