mirror of
https://github.com/Syfaro/fuzzysearch.git
synced 2024-11-05 06:23:08 +00:00
Add Weasyl metrics, rework metric names.
This commit is contained in:
parent
9920fff69c
commit
914a9b24ba
2
Cargo.lock
generated
2
Cargo.lock
generated
@ -1197,6 +1197,8 @@ dependencies = [
|
||||
"fuzzysearch-common",
|
||||
"image",
|
||||
"img_hash",
|
||||
"lazy_static",
|
||||
"prometheus",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
@ -1,6 +1,8 @@
|
||||
use anyhow::Context;
|
||||
use lazy_static::lazy_static;
|
||||
use prometheus::{register_histogram, register_int_gauge, Histogram, IntGauge};
|
||||
use prometheus::{
|
||||
register_histogram, register_int_gauge, Histogram, HistogramOpts, IntGauge, Opts,
|
||||
};
|
||||
use sqlx::Connection;
|
||||
use tracing_unwrap::ResultExt;
|
||||
|
||||
@ -9,20 +11,23 @@ use fuzzysearch_common::faktory::FaktoryClient;
|
||||
static USER_AGENT: &str = "e621-watcher / FuzzySearch Ingester / Syfaro <syfaro@huefox.com>";
|
||||
|
||||
lazy_static! {
|
||||
static ref SUBMISSION_BACKLOG: IntGauge = register_int_gauge!(
|
||||
"fuzzysearch_watcher_e621_submission_backlog",
|
||||
static ref SUBMISSION_BACKLOG: IntGauge = register_int_gauge!(Opts::new(
|
||||
"fuzzysearch_watcher_submission_backlog",
|
||||
"Number of submissions behind the latest ID"
|
||||
)
|
||||
.const_label("site", "e621"))
|
||||
.unwrap_or_log();
|
||||
static ref INDEX_DURATION: Histogram = register_histogram!(
|
||||
"fuzzysearch_watcher_e621_index_duration",
|
||||
static ref INDEX_DURATION: Histogram = register_histogram!(HistogramOpts::new(
|
||||
"fuzzysearch_watcher_index_duration_seconds",
|
||||
"Duration to load an index of submissions"
|
||||
)
|
||||
.const_label("site", "e621"))
|
||||
.unwrap_or_log();
|
||||
static ref SUBMISSION_DURATION: Histogram = register_histogram!(
|
||||
"fuzzysearch_watcher_e621_submission_duration",
|
||||
"Duration to ingest a submission"
|
||||
static ref SUBMISSION_DURATION: Histogram = register_histogram!(HistogramOpts::new(
|
||||
"fuzzysearch_watcher_submission_duration_seconds",
|
||||
"Duration to load an index of submissions"
|
||||
)
|
||||
.const_label("site", "e621"))
|
||||
.unwrap_or_log();
|
||||
}
|
||||
|
||||
|
@ -1,18 +1,38 @@
|
||||
use lazy_static::lazy_static;
|
||||
use prometheus::{
|
||||
register_counter, register_histogram, register_int_gauge_vec, Counter, Histogram,
|
||||
HistogramOpts, IntGaugeVec, Opts,
|
||||
};
|
||||
use tokio_postgres::Client;
|
||||
use tracing_unwrap::{OptionExt, ResultExt};
|
||||
|
||||
use fuzzysearch_common::faktory::FaktoryClient;
|
||||
|
||||
lazy_static! {
|
||||
static ref SUBMISSION_DURATION: prometheus::Histogram = prometheus::register_histogram!(
|
||||
"fuzzysearch_watcher_fa_processing_seconds",
|
||||
"Duration to process a submission"
|
||||
static ref INDEX_DURATION: Histogram = register_histogram!(HistogramOpts::new(
|
||||
"fuzzysearch_watcher_index_duration_seconds",
|
||||
"Duration to load an index of submissions"
|
||||
)
|
||||
.const_label("site", "furaffinity"))
|
||||
.unwrap_or_log();
|
||||
static ref USERS_ONLINE: prometheus::IntGaugeVec = prometheus::register_int_gauge_vec!(
|
||||
"fuzzysearch_watcher_fa_users_online_count",
|
||||
"Number of users online for each category",
|
||||
static ref SUBMISSION_DURATION: Histogram = register_histogram!(HistogramOpts::new(
|
||||
"fuzzysearch_watcher_submission_duration_seconds",
|
||||
"Duration to load an index of submissions"
|
||||
)
|
||||
.const_label("site", "furaffinity"))
|
||||
.unwrap_or_log();
|
||||
static ref SUBMISSION_MISSING: Counter = register_counter!(Opts::new(
|
||||
"fuzzysearch_watcher_submission_missing_total",
|
||||
"Number of submissions that were missing"
|
||||
)
|
||||
.const_label("site", "furaffinity"))
|
||||
.unwrap_or_log();
|
||||
static ref USERS_ONLINE: IntGaugeVec = register_int_gauge_vec!(
|
||||
Opts::new(
|
||||
"fuzzysearch_watcher_users_online",
|
||||
"Number of users online for each category"
|
||||
)
|
||||
.const_label("site", "furaffinity"),
|
||||
&["group"]
|
||||
)
|
||||
.unwrap_or_log();
|
||||
@ -176,6 +196,7 @@ async fn process_submission(
|
||||
Err(err) => {
|
||||
tracing::error!("Failed to load submission: {:?}", err);
|
||||
_timer.stop_and_discard();
|
||||
SUBMISSION_MISSING.inc();
|
||||
insert_null_submission(client, id).await.unwrap_or_log();
|
||||
return;
|
||||
}
|
||||
@ -186,6 +207,7 @@ async fn process_submission(
|
||||
None => {
|
||||
tracing::warn!("Submission did not exist");
|
||||
_timer.stop_and_discard();
|
||||
SUBMISSION_MISSING.inc();
|
||||
insert_null_submission(client, id).await.unwrap_or_log();
|
||||
return;
|
||||
}
|
||||
@ -272,11 +294,13 @@ async fn main() {
|
||||
tracing::info!("Started");
|
||||
|
||||
loop {
|
||||
let duration = INDEX_DURATION.start_timer();
|
||||
tracing::debug!("Fetching latest ID... ");
|
||||
let latest_id = fa
|
||||
.latest_id()
|
||||
.await
|
||||
.expect_or_log("Unable to get latest id");
|
||||
duration.stop_and_record();
|
||||
tracing::info!(latest_id = latest_id.0, "Got latest ID");
|
||||
|
||||
let online = latest_id.1;
|
||||
|
@ -10,6 +10,9 @@ anyhow = "1"
|
||||
tracing = "0.1"
|
||||
tracing-unwrap = "0.9"
|
||||
|
||||
prometheus = "0.12"
|
||||
lazy_static = "1"
|
||||
|
||||
reqwest = { version = "0.11", features = ["json"] }
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
|
||||
|
@ -1,9 +1,31 @@
|
||||
use prometheus::{register_counter, register_histogram, Counter, Histogram, HistogramOpts, Opts};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use sha2::{Digest, Sha256};
|
||||
use tracing_unwrap::{OptionExt, ResultExt};
|
||||
|
||||
use fuzzysearch_common::faktory::FaktoryClient;
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref INDEX_DURATION: Histogram = register_histogram!(HistogramOpts::new(
|
||||
"fuzzysearch_watcher_index_duration_seconds",
|
||||
"Duration to load an index of submissions"
|
||||
)
|
||||
.const_label("site", "weasyl"))
|
||||
.unwrap_or_log();
|
||||
static ref SUBMISSION_DURATION: Histogram = register_histogram!(HistogramOpts::new(
|
||||
"fuzzysearch_watcher_submission_duration_seconds",
|
||||
"Duration to load an index of submissions"
|
||||
)
|
||||
.const_label("site", "weasyl"))
|
||||
.unwrap_or_log();
|
||||
static ref SUBMISSION_MISSING: Counter = register_counter!(Opts::new(
|
||||
"fuzzysearch_watcher_submission_missing_total",
|
||||
"Number of submissions that were missing"
|
||||
)
|
||||
.const_label("site", "weasyl"))
|
||||
.unwrap_or_log();
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
struct WeasylMediaSubmission {
|
||||
#[serde(rename = "mediaid")]
|
||||
@ -233,7 +255,9 @@ async fn main() {
|
||||
.id
|
||||
.unwrap_or_default();
|
||||
|
||||
let duration = INDEX_DURATION.start_timer();
|
||||
let max = load_frontpage(&client, &api_key).await.unwrap_or_log();
|
||||
duration.stop_and_record();
|
||||
|
||||
tracing::info!(min, max, "Calculated range of submissions to check");
|
||||
|
||||
@ -246,13 +270,22 @@ async fn main() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let duration = SUBMISSION_DURATION.start_timer();
|
||||
|
||||
match load_submission(&client, &api_key, id).await.unwrap_or_log() {
|
||||
(Some(sub), json) => {
|
||||
process_submission(&pool, &client, &faktory, json, sub, &download_folder)
|
||||
.await
|
||||
.unwrap_or_log()
|
||||
.unwrap_or_log();
|
||||
|
||||
duration.stop_and_record();
|
||||
}
|
||||
(None, body) => {
|
||||
insert_null(&pool, body, id).await.unwrap_or_log();
|
||||
|
||||
SUBMISSION_MISSING.inc();
|
||||
duration.stop_and_discard();
|
||||
}
|
||||
(None, body) => insert_null(&pool, body, id).await.unwrap_or_log(),
|
||||
}
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user