anthrove/fuzzysearch
4
0
Fork 0
mirror of https://github.com/Syfaro/fuzzysearch.git synced 2024-11-05 06:23:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

More logging updates.

Browse Source
This commit is contained in:
Syfaro 2021-04-21 21:46:10 -04:00
parent 32fbfe1d00
commit 47703694d1
11 changed files with 65 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
Cargo.lock generated
View File

@ -131,9 +131,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a"
[[package]]
name = "backtrace"
version = "0.3.56"
version = "0.3.57"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9d117600f438b1707d4e4ae15d3595657288f8235a0eb593e80ecc98ab34e1bc"
checksum = "78ed203b9ba68b242c62b3fb7480f589dd49829be1edb3fe8fc8b4ffda2dcb8d"
dependencies = [
"addr2line",
"cfg-if 1.0.0",
@ -923,6 +923,7 @@ dependencies = [
"tempfile",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
@ -944,7 +945,7 @@ dependencies = [
"sqlx",
"tokio",
"tracing",
"tracing-subscriber",
"tracing-unwrap",
]
[[package]]
@ -969,7 +970,6 @@ dependencies = [
"tokio",
"tokio-postgres",
"tracing",
"tracing-subscriber",
"tracing-unwrap",
]
@ -988,7 +988,7 @@ dependencies = [
"sqlx",
"tokio",
"tracing",
"tracing-subscriber",
"tracing-unwrap",
]
[[package]]
@ -1004,7 +1004,6 @@ dependencies = [
"serde_json",
"thiserror",
"tracing",
"tracing-subscriber",
"tracing-unwrap",
]
@ -1257,15 +1256,15 @@ checksum = "4a1ce40d6fc9764887c2fdc7305c3dcc429ba11ff981c1509416afd5697e4437"
[[package]]
name = "httpdate"
version = "0.3.2"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47"
checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9"
[[package]]
name = "hyper"
version = "0.14.5"
version = "0.14.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8bf09f61b52cfcf4c00de50df88ae423d6c02354e385a86341133b5338630ad1"
checksum = "5f006b8784cfb01fe7aa9c46f5f5cd4cf5c85a8c612a0653ec97642979062665"
dependencies = [
"bytes",
"futures-channel",
@ -2947,9 +2946,9 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2"
[[package]]
name = "syn"
version = "1.0.69"
version = "1.0.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48fe99c6bd8b1cc636890bcc071842de909d902c81ac7dab53ba33c421ab8ffb"
checksum = "b9505f307c872bab8eb46f77ae357c8eba1fdacead58ee5a850116b1d7f82883"
dependencies = [
"proc-macro2",
"quote",

1
fuzzysearch-common/Cargo.toml
View File

@ -13,6 +13,7 @@ queue = ["faktory", "tokio", "serde_json"]
[dependencies]
anyhow = "1"
tracing = "0.1"
tracing-subscriber = "0.2"
serde = { version = "1", features = ["derive"] }
base64 = "0.13"

13
fuzzysearch-common/src/lib.rs
View File

@ -14,3 +14,16 @@ pub fn get_hasher() -> img_hash::Hasher<[u8; 8]> {
.preproc_dct()
.to_hasher()
}
/// Initialize the logger. This should only be called by the running binary.
pub fn init_logger() {
if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_env_filter(tracing_subscriber::EnvFilter::from_default_env())
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
}

2
fuzzysearch-ingest-e621/Cargo.toml
View File

@ -21,7 +21,7 @@ img_hash = "3"
sha2 = "0.9"
tracing = "0.1"
tracing-subscriber = "0.2"
tracing-unwrap = "0.9"
anyhow = "1"

45
fuzzysearch-ingest-e621/src/main.rs
View File

@ -2,6 +2,7 @@ use anyhow::Context;
use lazy_static::lazy_static;
use prometheus::{register_histogram, register_int_gauge, Histogram, IntGauge};
use sqlx::Connection;
use tracing_unwrap::ResultExt;
use fuzzysearch_common::faktory::FaktoryClient;
@ -12,50 +13,44 @@ lazy_static! {
"fuzzysearch_watcher_e621_submission_backlog",
"Number of submissions behind the latest ID"
)
.unwrap();
.unwrap_or_log();
static ref INDEX_DURATION: Histogram = register_histogram!(
"fuzzysearch_watcher_e621_index_duration",
"Duration to load an index of submissions"
)
.unwrap();
.unwrap_or_log();
static ref SUBMISSION_DURATION: Histogram = register_histogram!(
"fuzzysearch_watcher_e621_submission_duration",
"Duration to ingest a submission"
)
.unwrap();
.unwrap_or_log();
}
type Auth = (String, Option<String>);
#[tokio::main]
async fn main() -> anyhow::Result<()> {
if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
fuzzysearch_common::init_logger();
create_metrics_server().await;
let login = std::env::var("E621_LOGIN").expect("Missing E621_LOGIN");
let api_key = std::env::var("E621_API_KEY").expect("Missing E621_API_KEY");
let login = std::env::var("E621_LOGIN").expect_or_log("Missing E621_LOGIN");
let api_key = std::env::var("E621_API_KEY").expect_or_log("Missing E621_API_KEY");
let auth = (login, Some(api_key));
let client = reqwest::ClientBuilder::default()
.user_agent(USER_AGENT)
.build()?;
let mut conn =
sqlx::PgConnection::connect(&std::env::var("DATABASE_URL").expect("Missing DATABASE_URL"))
.await?;
let mut conn = sqlx::PgConnection::connect(
&std::env::var("DATABASE_URL").expect_or_log("Missing DATABASE_URL"),
)
.await?;
let faktory_dsn = std::env::var("FAKTORY_URL").expect("Missing FAKTORY_URL");
let faktory_dsn = std::env::var("FAKTORY_URL").expect_or_log("Missing FAKTORY_URL");
let faktory = FaktoryClient::connect(faktory_dsn)
.await
.expect("Unable to connect to Faktory");
.expect_or_log("Unable to connect to Faktory");
let max_id: i32 = sqlx::query!("SELECT max(id) max FROM e621")
.fetch_one(&mut conn)
@ -79,7 +74,7 @@ async fn main() -> anyhow::Result<()> {
let _hist = INDEX_DURATION.start_timer();
let lid = get_latest_id(&client, &auth)
.await
.expect("Unable to get latest ID");
.expect_or_log("Unable to get latest ID");
drop(_hist);
latest_id = Some(lid);
@ -100,7 +95,7 @@ async fn main() -> anyhow::Result<()> {
min_id = match post_ids.iter().max() {
Some(id) => *id,
None => {
tracing::warn!("Found no new posts, sleeping");
tracing::info!("Found no new posts, sleeping");
tokio::time::sleep(std::time::Duration::from_secs(60 * 5)).await;
continue;
}
@ -359,7 +354,9 @@ async fn provide_metrics(
let encoder = TextEncoder::new();
let metric_families = prometheus::gather();
encoder.encode(&metric_families, &mut buffer).unwrap();
encoder
.encode(&metric_families, &mut buffer)
.unwrap_or_log();
Ok(Response::new(Body::from(buffer)))
}
@ -376,11 +373,11 @@ async fn create_metrics_server() {
make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(provide_metrics)) });
let addr: SocketAddr = std::env::var("METRICS_HOST")
.expect("Missing METRICS_HOST")
.expect_or_log("Missing METRICS_HOST")
.parse()
.expect("Invalid METRICS_HOST");
.expect_or_log("Invalid METRICS_HOST");
let server = Server::bind(&addr).serve(make_svc);
tokio::spawn(async move { server.await.expect("Metrics server error") });
tokio::spawn(async move { server.await.expect_or_log("Metrics server error") });
}

1
fuzzysearch-ingest-furaffinity/Cargo.toml
View File

@ -17,7 +17,6 @@ prometheus = { version = "0.12", features = ["process"] }
lazy_static = "1"
futures-retry = "0.6"
tracing = "0.1"
tracing-subscriber = "0.2"
tracing-unwrap = "0.9"
faktory = "0.11"
anyhow = "1"

9
fuzzysearch-ingest-furaffinity/src/main.rs
View File

@ -270,14 +270,7 @@ async fn process_submission(
#[tokio::main]
async fn main() {
if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
fuzzysearch_common::init_logger();
let (cookie_a, cookie_b) = (
std::env::var("FA_A").expect_or_log("Missing FA_A"),

2
fuzzysearch-ingest-weasyl/Cargo.toml
View File

@ -8,7 +8,7 @@ edition = "2018"
anyhow = "1"
tracing = "0.1"
tracing-subscriber = "0.2"
tracing-unwrap = "0.9"
reqwest = { version = "0.11", features = ["json"] }
tokio = { version = "1", features = ["full"] }

36
fuzzysearch-ingest-weasyl/src/main.rs
View File

@ -1,5 +1,6 @@
use serde::{Deserialize, Serialize};
use sha2::{Digest, Sha256};
use tracing_unwrap::{OptionExt, ResultExt};
use fuzzysearch_common::faktory::FaktoryClient;
@ -130,7 +131,7 @@ async fn process_submission(
tracing::debug!("Processing submission");
let data = client
.get(&sub.media.submission.first().unwrap().url)
.get(&sub.media.submission.first().unwrap_or_log().url)
.send()
.await?
.bytes()
@ -158,7 +159,7 @@ async fn process_submission(
site: fuzzysearch_common::types::Site::Weasyl,
site_id: sub.id,
artist: sub.owner_login.clone(),
file_url: sub.media.submission.first().unwrap().url.clone(),
file_url: sub.media.submission.first().unwrap_or_log().url.clone(),
file_sha256: Some(result.to_vec()),
hash: num.map(|hash| hash.to_be_bytes()),
})
@ -195,39 +196,32 @@ async fn insert_null(
#[tokio::main]
async fn main() {
if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
fuzzysearch_common::init_logger();
let api_key = std::env::var("WEASYL_APIKEY").unwrap();
let api_key = std::env::var("WEASYL_APIKEY").unwrap_or_log();
let pool = sqlx::postgres::PgPoolOptions::new()
.max_connections(2)
.connect(&std::env::var("DATABASE_URL").unwrap())
.connect(&std::env::var("DATABASE_URL").unwrap_or_log())
.await
.unwrap();
.unwrap_or_log();
let client = reqwest::Client::new();
let faktory_dsn = std::env::var("FAKTORY_URL").expect("Missing FAKTORY_URL");
let faktory_dsn = std::env::var("FAKTORY_URL").expect_or_log("Missing FAKTORY_URL");
let faktory = FaktoryClient::connect(faktory_dsn)
.await
.expect("Unable to connect to Faktory");
.expect_or_log("Unable to connect to Faktory");
loop {
let min = sqlx::query!("SELECT max(id) id FROM weasyl")
.fetch_one(&pool)
.await
.unwrap()
.unwrap_or_log()
.id
.unwrap_or_default();
let max = load_frontpage(&client, &api_key).await.unwrap();
let max = load_frontpage(&client, &api_key).await.unwrap_or_log();
tracing::info!(min, max, "Calculated range of submissions to check");
@ -235,16 +229,16 @@ async fn main() {
let row: Option<_> = sqlx::query!("SELECT id FROM weasyl WHERE id = $1", id)
.fetch_optional(&pool)
.await
.unwrap();
.unwrap_or_log();
if row.is_some() {
continue;
}
match load_submission(&client, &api_key, id).await.unwrap() {
match load_submission(&client, &api_key, id).await.unwrap_or_log() {
(Some(sub), json) => process_submission(&pool, &client, &faktory, json, sub)
.await
.unwrap(),
(None, body) => insert_null(&pool, body, id).await.unwrap(),
.unwrap_or_log(),
(None, body) => insert_null(&pool, body, id).await.unwrap_or_log(),
}
}

1
fuzzysearch-webhook/Cargo.toml
View File

@ -6,7 +6,6 @@ edition = "2018"
[dependencies]
tracing = "0.1"
tracing-subscriber = "0.2"
tracing-unwrap = "0.9"
thiserror = "1"

9
fuzzysearch-webhook/src/main.rs
View File

@ -25,14 +25,7 @@ pub enum WebhookError {
}
fn main() {
if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) {
tracing_subscriber::fmt::Subscriber::builder()
.json()
.with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339())
.init();
} else {
tracing_subscriber::fmt::init();
}
fuzzysearch_common::init_logger();
tracing::info!("Starting...");