JSON log output (#9)

* Allow outputting JSON logs from ingesters.

* Skip instrument for faktory.

* Use more specific timestamp format.

* Update logger for webhooks too.

Cargo.lock

@@ -78,9 +78,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
 
 [[package]]
 name = "async-stream"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c"
+checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e"
 dependencies = [
  "async-stream-impl",
  "futures-core",
@@ -88,9 +88,9 @@ dependencies = [
 
 [[package]]
 name = "async-stream-impl"
-version = "0.3.0"
+version = "0.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70"
+checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -258,9 +258,9 @@ checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8"
 
 [[package]]
 name = "build_const"
-version = "0.2.1"
+version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39"
+checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7"
 
 [[package]]
 name = "bumpalo"
@@ -987,6 +987,8 @@ dependencies = [
  "sha2 0.9.3",
  "sqlx",
  "tokio",
+ "tracing",
+ "tracing-subscriber",
 ]
 
 [[package]]
@@ -1468,9 +1470,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "lexical-core"
-version = "0.7.5"
+version = "0.7.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374"
+checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
 dependencies = [
  "arrayvec",
  "bitflags",
@@ -1527,16 +1529,13 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
 
 [[package]]
 name = "markup5ever"
-version = "0.10.0"
+version = "0.10.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "aae38d669396ca9b707bfc3db254bc382ddb94f57cc5c235f34623a669a01dab"
+checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"
 dependencies = [
  "log",
  "phf",
  "phf_codegen",
- "serde",
- "serde_derive",
- "serde_json",
  "string_cache",
  "string_cache_codegen",
  "tendril",
@@ -2756,9 +2755,9 @@ checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27"
 
 [[package]]
 name = "slab"
-version = "0.4.2"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
+checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
 
 [[package]]
 name = "smallvec"

fuzzysearch-ingest-e621/src/main.rs

@@ -29,7 +29,14 @@ type Auth = (String, Option<String>);
 
 #[tokio::main]
 async fn main() -> anyhow::Result<()> {
-    tracing_subscriber::fmt::init();
+    if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
+        tracing_subscriber::fmt::Subscriber::builder()
+            .json()
+            .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
+            .init();
+    } else {
+        tracing_subscriber::fmt::init();
+    }
 
     create_metrics_server().await;
 

fuzzysearch-ingest-furaffinity/src/main.rs

@@ -270,7 +270,14 @@ async fn process_submission(
 
 #[tokio::main]
 async fn main() {
-    tracing_subscriber::fmt::init();
+    if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
+        tracing_subscriber::fmt::Subscriber::builder()
+            .json()
+            .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
+            .init();
+    } else {
+        tracing_subscriber::fmt::init();
+    }
 
     let (cookie_a, cookie_b) = (
         std::env::var("FA_A").expect_or_log("Missing FA_A"),

fuzzysearch-ingest-weasyl/Cargo.toml

@@ -7,6 +7,9 @@ edition = "2018"
 [dependencies]
 anyhow = "1"
 
+tracing = "0.1"
+tracing-subscriber = "0.2"
+
 reqwest = { version = "0.11", features = ["json"] }
 tokio = { version = "1", features = ["full"] }
 

fuzzysearch-ingest-weasyl/src/main.rs

@@ -75,12 +75,13 @@ async fn load_frontpage(client: &reqwest::Client, api_key: &str) -> anyhow::Resu
     Ok(max as i32)
 }
 
+#[tracing::instrument(skip(client, api_key))]
 async fn load_submission(
     client: &reqwest::Client,
     api_key: &str,
     id: i32,
 ) -> anyhow::Result<(Option<WeasylSubmission>, serde_json::Value)> {
-    println!("Loading submission {}", id);
+    tracing::debug!("Loading submission");
 
     let body: serde_json::Value = client
         .get(&format!(
@@ -114,6 +115,7 @@ async fn load_submission(
     Ok((res, body))
 }
 
+#[tracing::instrument(skip(pool, client, faktory, body, sub), fields(id = sub.id))]
 async fn process_submission(
     pool: &sqlx::Pool<sqlx::Postgres>,
     client: &reqwest::Client,
@@ -121,7 +123,7 @@ async fn process_submission(
     body: serde_json::Value,
     sub: WeasylSubmission,
 ) -> anyhow::Result<()> {
-    println!("Processing submission {}", sub.id);
+    tracing::debug!("Processing submission");
 
     let data = client
         .get(&sub.media.submission.first().unwrap().url)
@@ -138,7 +140,7 @@ async fn process_submission(
         let num = i64::from_be_bytes(bytes);
         Some(num)
     } else {
-        println!("Unable to decode image on submission {}", sub.id);
+        tracing::warn!("Unable to decode image");
 
         None
     };
@@ -172,12 +174,13 @@ async fn process_submission(
     Ok(())
 }
 
+#[tracing::instrument(skip(pool, body))]
 async fn insert_null(
     pool: &sqlx::Pool<sqlx::Postgres>,
     body: serde_json::Value,
     id: i32,
 ) -> anyhow::Result<()> {
-    println!("Inserting null for submission {}", id);
+    tracing::debug!("Inserting null submission");
 
     sqlx::query!("INSERT INTO WEASYL (id, data) VALUES ($1, $2)", id, body)
         .execute(pool)
@@ -188,6 +191,15 @@ async fn insert_null(
 
 #[tokio::main]
 async fn main() {
+    if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
+        tracing_subscriber::fmt::Subscriber::builder()
+            .json()
+            .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
+            .init();
+    } else {
+        tracing_subscriber::fmt::init();
+    }
+
     let api_key = std::env::var("WEASYL_APIKEY").unwrap();
 
     let pool = sqlx::postgres::PgPoolOptions::new()

fuzzysearch-webhook/src/main.rs

@@ -25,7 +25,14 @@ pub enum WebhookError {
 }
 
 fn main() {
-    tracing_subscriber::fmt::init();
+    if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
+        tracing_subscriber::fmt::Subscriber::builder()
+            .json()
+            .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
+            .init();
+    } else {
+        tracing_subscriber::fmt::init();
+    }
 
     tracing::info!("Starting...");