JSON log output (#9)

* Allow outputting JSON logs from ingesters.

* Skip instrument for faktory.

* Use more specific timestamp format.

* Update logger for webhooks too.
This commit is contained in:
Syfaro 2021-04-21 19:26:15 -04:00 committed by GitHub
parent dd7805d052
commit 3974f85ce8
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 57 additions and 22 deletions

29
Cargo.lock generated
View File

@ -78,9 +78,9 @@ checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]] [[package]]
name = "async-stream" name = "async-stream"
version = "0.3.0" version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e"
dependencies = [ dependencies = [
"async-stream-impl", "async-stream-impl",
"futures-core", "futures-core",
@ -88,9 +88,9 @@ dependencies = [
[[package]] [[package]]
name = "async-stream-impl" name = "async-stream-impl"
version = "0.3.0" version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340"
dependencies = [ dependencies = [
"proc-macro2", "proc-macro2",
"quote", "quote",
@ -258,9 +258,9 @@ checksum = "40e38929add23cdf8a366df9b0e088953150724bcbe5fc330b0d8eb3b328eec8"
[[package]] [[package]]
name = "build_const" name = "build_const"
version = "0.2.1" version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7"
[[package]] [[package]]
name = "bumpalo" name = "bumpalo"
@ -987,6 +987,8 @@ dependencies = [
"sha2 0.9.3", "sha2 0.9.3",
"sqlx", "sqlx",
"tokio", "tokio",
"tracing",
"tracing-subscriber",
] ]
[[package]] [[package]]
@ -1468,9 +1470,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]] [[package]]
name = "lexical-core" name = "lexical-core"
version = "0.7.5" version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" checksum = "6607c62aa161d23d17a9072cc5da0be67cdfc89d3afb1e8d9c842bebc2525ffe"
dependencies = [ dependencies = [
"arrayvec", "arrayvec",
"bitflags", "bitflags",
@ -1527,16 +1529,13 @@ checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
[[package]] [[package]]
name = "markup5ever" name = "markup5ever"
version = "0.10.0" version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae38d669396ca9b707bfc3db254bc382ddb94f57cc5c235f34623a669a01dab" checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"
dependencies = [ dependencies = [
"log", "log",
"phf", "phf",
"phf_codegen", "phf_codegen",
"serde",
"serde_derive",
"serde_json",
"string_cache", "string_cache",
"string_cache_codegen", "string_cache_codegen",
"tendril", "tendril",
@ -2756,9 +2755,9 @@ checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27"
[[package]] [[package]]
name = "slab" name = "slab"
version = "0.4.2" version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"
[[package]] [[package]]
name = "smallvec" name = "smallvec"

View File

@ -29,7 +29,14 @@ type Auth = (String, Option<String>);
#[tokio::main] #[tokio::main]
async fn main() -> anyhow::Result<()> { async fn main() -> anyhow::Result<()> {
tracing_subscriber::fmt::init(); if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
create_metrics_server().await; create_metrics_server().await;

View File

@ -270,7 +270,14 @@ async fn process_submission(
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
tracing_subscriber::fmt::init(); if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
let (cookie_a, cookie_b) = ( let (cookie_a, cookie_b) = (
std::env::var("FA_A").expect_or_log("Missing FA_A"), std::env::var("FA_A").expect_or_log("Missing FA_A"),

View File

@ -7,6 +7,9 @@ edition = "2018"
[dependencies] [dependencies]
anyhow = "1" anyhow = "1"
tracing = "0.1"
tracing-subscriber = "0.2"
reqwest = { version = "0.11", features = ["json"] } reqwest = { version = "0.11", features = ["json"] }
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }

View File

@ -75,12 +75,13 @@ async fn load_frontpage(client: &reqwest::Client, api_key: &str) -> anyhow::Resu
Ok(max as i32) Ok(max as i32)
} }
#[tracing::instrument(skip(client, api_key))]
async fn load_submission( async fn load_submission(
client: &reqwest::Client, client: &reqwest::Client,
api_key: &str, api_key: &str,
id: i32, id: i32,
) -> anyhow::Result<(Option<WeasylSubmission>, serde_json::Value)> { ) -> anyhow::Result<(Option<WeasylSubmission>, serde_json::Value)> {
println!("Loading submission {}", id); tracing::debug!("Loading submission");
let body: serde_json::Value = client let body: serde_json::Value = client
.get(&format!( .get(&format!(
@ -114,6 +115,7 @@ async fn load_submission(
Ok((res, body)) Ok((res, body))
} }
#[tracing::instrument(skip(pool, client, faktory, body, sub), fields(id = sub.id))]
async fn process_submission( async fn process_submission(
pool: &sqlx::Pool<sqlx::Postgres>, pool: &sqlx::Pool<sqlx::Postgres>,
client: &reqwest::Client, client: &reqwest::Client,
@ -121,7 +123,7 @@ async fn process_submission(
body: serde_json::Value, body: serde_json::Value,
sub: WeasylSubmission, sub: WeasylSubmission,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
println!("Processing submission {}", sub.id); tracing::debug!("Processing submission");
let data = client let data = client
.get(&sub.media.submission.first().unwrap().url) .get(&sub.media.submission.first().unwrap().url)
@ -138,7 +140,7 @@ async fn process_submission(
let num = i64::from_be_bytes(bytes); let num = i64::from_be_bytes(bytes);
Some(num) Some(num)
} else { } else {
println!("Unable to decode image on submission {}", sub.id); tracing::warn!("Unable to decode image");
None None
}; };
@ -172,12 +174,13 @@ async fn process_submission(
Ok(()) Ok(())
} }
#[tracing::instrument(skip(pool, body))]
async fn insert_null( async fn insert_null(
pool: &sqlx::Pool<sqlx::Postgres>, pool: &sqlx::Pool<sqlx::Postgres>,
body: serde_json::Value, body: serde_json::Value,
id: i32, id: i32,
) -> anyhow::Result<()> { ) -> anyhow::Result<()> {
println!("Inserting null for submission {}", id); tracing::debug!("Inserting null submission");
sqlx::query!("INSERT INTO WEASYL (id, data) VALUES ($1, $2)", id, body) sqlx::query!("INSERT INTO WEASYL (id, data) VALUES ($1, $2)", id, body)
.execute(pool) .execute(pool)
@ -188,6 +191,15 @@ async fn insert_null(
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
let api_key = std::env::var("WEASYL_APIKEY").unwrap(); let api_key = std::env::var("WEASYL_APIKEY").unwrap();
let pool = sqlx::postgres::PgPoolOptions::new() let pool = sqlx::postgres::PgPoolOptions::new()

View File

@ -25,7 +25,14 @@ pub enum WebhookError {
} }
fn main() { fn main() {
tracing_subscriber::fmt::init(); if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
tracing::info!("Starting..."); tracing::info!("Starting...");