Making custom Elsticsearch logger.

src/logging/core.rs

@@ -94,7 +94,7 @@ impl Append for StructuredChannelAppender {
 
     fn flush(&self) {}
 }
-pub fn log_builder(conf: &AppConfig, location: &Option<String>) {
+pub async fn log_builder(conf: &AppConfig, location: &Option<String>) {
     let log_level = match conf.log_level.as_str() {
         "info" => LevelFilter::Info,
         "error" => LevelFilter::Error,
@@ -118,7 +118,7 @@ pub fn log_builder(conf: &AppConfig, location: &Option<String>) {
     if let Some(backend) = conf.log_structured.as_deref() {
         if let Some(backend_name) = backend.split_whitespace().next() {
             let _ = set_backend(backend_name.to_string());
-            init_structured_log();
+            init_structured_log().await;
         }
         let pingora_appender: Option<Box<dyn Append>> = PINGORA_LOG_SENDER
             .get()
@@ -215,7 +215,7 @@ pub fn log_builder(conf: &AppConfig, location: &Option<String>) {
     }
 }
 
-pub fn init_access_log(level_str: &str) {
+pub async fn init_access_log(level_str: &str) {
     let level = LogLevel::from_str(level_str);
     let _ = ACCESS_LOG.set(level);
 }
@@ -255,7 +255,7 @@ impl MatchStatus {
     }
 }
 
-pub fn access_log(response_code: u16, summary: &str, session: &Session) {
+pub async fn access_log(response_code: u16, summary: &str, session: &Session) {
     let level = ACCESS_LOG.get().unwrap_or(&LogLevel::None);
     let status = MatchStatus::from_code(response_code);
 
@@ -287,7 +287,7 @@ pub fn access_log(response_code: u16, summary: &str, session: &Session) {
     };
 
     if IS_STRUCTURED.get().is_some() {
-        write_access_log(&msg);
+        write_access_log(&msg).await;
         return;
     }
     if let Some(sender) = LOG_SENDER.get() {
@@ -297,37 +297,44 @@ pub fn access_log(response_code: u16, summary: &str, session: &Session) {
     }
 }
 
-pub fn init_access_logging(enabled: Option<String>) {
+pub async fn init_access_logging(enabled: Option<String>) {
     if enabled.is_some() {
         LOGGING_ERRORS.set(0);
         info!("Enabling {:?} log, with buffer of {} messages", ACCESS_LOG.get().unwrap_or(&LogLevel::None), LOG_BUFFER);
         let (ltx, lrx) = mpsc::channel(LOG_BUFFER);
         let _ = LOG_SENDER.set(ltx);
-        std::thread::spawn(move || access_log_receiver(lrx));
+        drop(tokio::spawn(async move { access_log_receiver(lrx).await }));
     }
 }
 
-pub fn init_structured_log() {
+pub async fn init_structured_log() {
     let (tx, mut rx) = mpsc::channel::<StructuredSystemLog>(LOG_BUFFER);
     let _ = PINGORA_LOG_SENDER.set(tx);
     let backend = get_backend();
-    std::thread::Builder::new()
-        .name("structured-log-receiver".to_string())
-        .spawn(move || {
-            while let Some(syslog) = rx.blocking_recv() {
-                sendlog(backend, &syslog);
-            }
-        })
-        .expect("Failed to spawn log thread");
+
+    drop(tokio::spawn(async move {
+        while let Some(syslog) = rx.recv().await {
+            sendlog(backend, &syslog).await;
+        }
+    }));
+
+    // std::thread::Builder::new()
+    //     .name("structured-log-receiver".to_string())
+    //     .spawn(move || {
+    //         while let Some(syslog) = rx.blocking_recv() {
+    //             sendlog(backend, &syslog);
+    //         }
+    //     })
+    //     .expect("Failed to spawn log thread");
 }
 
-pub fn access_log_receiver(mut receiver: mpsc::Receiver<LogMessage>) {
-    while let Some(msg) = receiver.blocking_recv() {
-        write_access_log(&msg);
+pub async fn access_log_receiver(mut receiver: mpsc::Receiver<LogMessage>) {
+    while let Some(msg) = receiver.recv().await {
+        write_access_log(&msg).await;
     }
 }
 
-fn write_access_log(msg: &LogMessage) {
+async fn write_access_log(msg: &LogMessage) {
     match MatchStatus::from_code(msg.response_code) {
         MatchStatus::Ok2xx => info!(
             "{}, {}, {}, client: {}, version: {:?}, useragent: {}",

src/logging/elasticsearch.rs

@@ -1,22 +1,30 @@
 use crate::logging::core::StructuredSystemLog;
 use crate::logging::types::{LogBackendPlugin, WriteLog};
+use crate::utils::hcclient::httpclient;
+use async_trait::async_trait;
+use bytes::Bytes;
 use std::env;
 use std::sync::LazyLock;
 
 pub struct ElasticSearchConfig {
-    pub hosts: &'static str,
+    pub hosts: &'static [&'static str],
     pub user: &'static str,
     pub password: &'static str,
 }
 
 impl ElasticSearchConfig {
     pub fn load_from_env() -> Self {
-        let host = env::var("LOG_ELASTIC_HOSTS").unwrap_or_else(|_| "127.0.0.1".to_string());
+        let hosts: Vec<&'static str> = env::var("LOG_ELASTIC_HOSTS")
+            .unwrap_or_else(|_| "127.0.0.1".to_string())
+            .split(',')
+            .map(|s| -> &'static str { Box::leak(s.trim().to_string().into_boxed_str()) })
+            .collect();
+
         let user = env::var("LOG_ELASTIC_USER").unwrap_or_else(|_| "elastic".to_string());
         let password = env::var("LOG_ELASTIC_PASSWORD").unwrap_or_else(|_| "elastic".to_string());
 
         Self {
-            hosts: Box::leak(host.into_boxed_str()),
+            hosts: Box::leak(hosts.into_boxed_slice()),
             user: Box::leak(user.into_boxed_str()),
             password: Box::leak(password.into_boxed_str()),
         }
@@ -25,13 +33,18 @@ impl ElasticSearchConfig {
 pub static ELASTIC: LazyLock<ElasticSearchConfig> = LazyLock::new(ElasticSearchConfig::load_from_env);
 pub struct ElasticSearch;
 
+#[async_trait]
 impl WriteLog for ElasticSearch {
-    fn writelog(&self, msg: &StructuredSystemLog) {
-        if let Ok(jsonmsg) = serde_json::to_string(&msg) {
-            // ElasticSearch logic is not yet implemented.
-            println!("{} - {}:{} => {}", ELASTIC.hosts, ELASTIC.user, ELASTIC.password, jsonmsg);
-            // let resp = httpclient(method, tls, host, path, upstream.address.as_ref(), upstream.port, link).await;
-        }
+    async fn writelog(&self, msg: &StructuredSystemLog) {
+        let payload = match serde_json::to_vec(&msg) {
+            Ok(vec) => Bytes::from(vec),
+            Err(e) => {
+                log::warn!("Failed to serialize structured system log to JSON bytes: {}", e);
+                return;
+            }
+        };
+        println!("{} => {} => {}:{:?}", ELASTIC.hosts[0], ELASTIC.hosts[1], ELASTIC.user, ELASTIC.password);
+        let _ = httpclient("POST", false, "127.0.0.1", "/i", "localhost", 8000, "http://127.0.0.1:8000/d".to_string(), payload).await;
     }
 }
 

src/logging/example.rs

@@ -1,10 +1,11 @@
 use crate::logging::core::StructuredSystemLog;
 use crate::logging::types::{LogBackendPlugin, WriteLog};
+use async_trait::async_trait;
 
 pub struct Example;
-
+#[async_trait]
 impl WriteLog for Example {
-    fn writelog(&self, msg: &StructuredSystemLog) {
+    async fn writelog(&self, msg: &StructuredSystemLog) {
         // Here comes the backend logic
         println!("Sending log Example : {:?}", msg);
     }

src/logging/types.rs

@@ -1,9 +1,11 @@
 use crate::logging::core::StructuredSystemLog;
+use async_trait::async_trait;
 use std::collections::HashMap;
 use std::sync::LazyLock;
 
+#[async_trait]
 pub trait WriteLog: Send + Sync {
-    fn writelog(&self, msg: &StructuredSystemLog);
+    async fn writelog(&self, msg: &StructuredSystemLog);
 }
 
 pub struct LogBackendPlugin {
@@ -21,9 +23,9 @@ static BACKENDS: LazyLock<HashMap<&'static str, Box<dyn WriteLog>>> = LazyLock::
     map
 });
 
-pub fn sendlog(backend: &str, msg: &StructuredSystemLog) {
+pub async fn sendlog(backend: &str, msg: &StructuredSystemLog) {
     if let Some(logger) = BACKENDS.get(backend) {
-        logger.writelog(msg);
+        logger.writelog(msg).await;
     } else {
         log::warn!("Unsupported logging mechanism: {}", backend);
     }