From 3974f85ce81947e4f45bd2220c42f4866c3e85c5 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Wed, 21 Apr 2021 19:26:15 -0400 Subject: [PATCH] JSON log output (#9) * Allow outputting JSON logs from ingesters. * Skip instrument for faktory. * Use more specific timestamp format. * Update logger for webhooks too. --- Cargo.lock | 29 +++++++++++----------- fuzzysearch-ingest-e621/src/main.rs | 9 ++++++- fuzzysearch-ingest-furaffinity/src/main.rs | 9 ++++++- fuzzysearch-ingest-weasyl/Cargo.toml | 3 +++ fuzzysearch-ingest-weasyl/src/main.rs | 20 ++++++++++++--- fuzzysearch-webhook/src/main.rs | 9 ++++++- 6 files changed, 57 insertions(+), 22 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index e35ca7e..2ec2fc9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -78,9 +78,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" [[package]] name = "async-stream" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e" dependencies = [ "async-stream-impl", "futures-core", @@ -88,9 +88,9 @@ dependencies = [ [[package]] name = "async-stream-impl" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340" dependencies = [ "proc-macro2", "quote", @@ -258,9 +258,9 @@ checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8" [[package]] name = "build_const" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" +checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" [[package]] name = "bumpalo" @@ -987,6 +987,8 @@ dependencies = [ "sha2 0.9.3", "sqlx", "tokio", + "tracing", + "tracing-subscriber", ] [[package]] @@ -1468,9 +1470,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" [[package]] name = "lexical-core" -version = "0.7.5" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" +checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe" dependencies = [ "arrayvec", "bitflags", @@ -1527,16 +1529,13 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" [[package]] name = "markup5ever" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae38d669396ca9b707bfc3db254bc382ddb94f57cc5c235f34623a669a01dab" +checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd" dependencies = [ "log", "phf", "phf_codegen", - "serde", - "serde_derive", - "serde_json", "string_cache", "string_cache_codegen", "tendril", @@ -2756,9 +2755,9 @@ checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" [[package]] name = "slab" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527" [[package]] name = "smallvec" diff --git a/fuzzysearch-ingest-e621/src/main.rs b/fuzzysearch-ingest-e621/src/main.rs index ed62cb6..608f4f0 100644 --- a/fuzzysearch-ingest-e621/src/main.rs +++ b/fuzzysearch-ingest-e621/src/main.rs @@ -29,7 +29,14 @@ type Auth = (String, Option); #[tokio::main] async fn main() -> anyhow::Result<()> { - tracing_subscriber::fmt::init(); + if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) { + tracing_subscriber::fmt::Subscriber::builder() + .json() + .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339()) + .init(); + } else { + tracing_subscriber::fmt::init(); + } create_metrics_server().await; diff --git a/fuzzysearch-ingest-furaffinity/src/main.rs b/fuzzysearch-ingest-furaffinity/src/main.rs index ac80c4d..9eab1cf 100644 --- a/fuzzysearch-ingest-furaffinity/src/main.rs +++ b/fuzzysearch-ingest-furaffinity/src/main.rs @@ -270,7 +270,14 @@ async fn process_submission( #[tokio::main] async fn main() { - tracing_subscriber::fmt::init(); + if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) { + tracing_subscriber::fmt::Subscriber::builder() + .json() + .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339()) + .init(); + } else { + tracing_subscriber::fmt::init(); + } let (cookie_a, cookie_b) = ( std::env::var("FA_A").expect_or_log("Missing FA_A"), diff --git a/fuzzysearch-ingest-weasyl/Cargo.toml b/fuzzysearch-ingest-weasyl/Cargo.toml index 2c5afbb..52e90ae 100644 --- a/fuzzysearch-ingest-weasyl/Cargo.toml +++ b/fuzzysearch-ingest-weasyl/Cargo.toml @@ -7,6 +7,9 @@ edition = "2018" [dependencies] anyhow = "1" +tracing = "0.1" +tracing-subscriber = "0.2" + reqwest = { version = "0.11", features = ["json"] } tokio = { version = "1", features = ["full"] } diff --git a/fuzzysearch-ingest-weasyl/src/main.rs b/fuzzysearch-ingest-weasyl/src/main.rs index 31e6654..0d8f075 100644 --- a/fuzzysearch-ingest-weasyl/src/main.rs +++ b/fuzzysearch-ingest-weasyl/src/main.rs @@ -75,12 +75,13 @@ async fn load_frontpage(client: &reqwest::Client, api_key: &str) -> anyhow::Resu Ok(max as i32) } +#[tracing::instrument(skip(client, api_key))] async fn load_submission( client: &reqwest::Client, api_key: &str, id: i32, ) -> anyhow::Result<(Option, serde_json::Value)> { - println!("Loading submission {}", id); + tracing::debug!("Loading submission"); let body: serde_json::Value = client .get(&format!( @@ -114,6 +115,7 @@ async fn load_submission( Ok((res, body)) } +#[tracing::instrument(skip(pool, client, faktory, body, sub), fields(id = sub.id))] async fn process_submission( pool: &sqlx::Pool, client: &reqwest::Client, @@ -121,7 +123,7 @@ async fn process_submission( body: serde_json::Value, sub: WeasylSubmission, ) -> anyhow::Result<()> { - println!("Processing submission {}", sub.id); + tracing::debug!("Processing submission"); let data = client .get(&sub.media.submission.first().unwrap().url) @@ -138,7 +140,7 @@ async fn process_submission( let num = i64::from_be_bytes(bytes); Some(num) } else { - println!("Unable to decode image on submission {}", sub.id); + tracing::warn!("Unable to decode image"); None }; @@ -172,12 +174,13 @@ async fn process_submission( Ok(()) } +#[tracing::instrument(skip(pool, body))] async fn insert_null( pool: &sqlx::Pool, body: serde_json::Value, id: i32, ) -> anyhow::Result<()> { - println!("Inserting null for submission {}", id); + tracing::debug!("Inserting null submission"); sqlx::query!("INSERT INTO WEASYL (id, data) VALUES ($1, $2)", id, body) .execute(pool) @@ -188,6 +191,15 @@ async fn insert_null( #[tokio::main] async fn main() { + if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) { + tracing_subscriber::fmt::Subscriber::builder() + .json() + .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339()) + .init(); + } else { + tracing_subscriber::fmt::init(); + } + let api_key = std::env::var("WEASYL_APIKEY").unwrap(); let pool = sqlx::postgres::PgPoolOptions::new() diff --git a/fuzzysearch-webhook/src/main.rs b/fuzzysearch-webhook/src/main.rs index 16d86e5..4a09f55 100644 --- a/fuzzysearch-webhook/src/main.rs +++ b/fuzzysearch-webhook/src/main.rs @@ -25,7 +25,14 @@ pub enum WebhookError { } fn main() { - tracing_subscriber::fmt::init(); + if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) { + tracing_subscriber::fmt::Subscriber::builder() + .json() + .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339()) + .init(); + } else { + tracing_subscriber::fmt::init(); + } tracing::info!("Starting...");