mirror of
https://github.com/Syfaro/fuzzysearch.git
synced 2024-11-10 17:02:38 +00:00
Tolerate and log hashing errors.
This commit is contained in:
parent
797fa4cb11
commit
f02a8db8c0
14
Cargo.lock
generated
14
Cargo.lock
generated
@ -59,7 +59,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "bb8"
|
name = "bb8"
|
||||||
version = "0.4.0-pre"
|
version = "0.4.0-pre"
|
||||||
source = "git+https://github.com/khuey/bb8#787b525621a27209da4fe5d0025c7e47a0ff9161"
|
source = "git+https://github.com/khuey/bb8.git#787b525621a27209da4fe5d0025c7e47a0ff9161"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
"async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
@ -69,10 +69,10 @@ dependencies = [
|
|||||||
[[package]]
|
[[package]]
|
||||||
name = "bb8-postgres"
|
name = "bb8-postgres"
|
||||||
version = "0.4.0-pre"
|
version = "0.4.0-pre"
|
||||||
source = "git+https://github.com/khuey/bb8#787b525621a27209da4fe5d0025c7e47a0ff9161"
|
source = "git+https://github.com/khuey/bb8.git#787b525621a27209da4fe5d0025c7e47a0ff9161"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
"async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"bb8 0.4.0-pre (git+https://github.com/khuey/bb8)",
|
"bb8 0.4.0-pre (git+https://github.com/khuey/bb8.git)",
|
||||||
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"tokio 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
"tokio 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"tokio-postgres 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
"tokio-postgres 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
@ -299,8 +299,8 @@ dependencies = [
|
|||||||
name = "e621-watcher"
|
name = "e621-watcher"
|
||||||
version = "0.1.0"
|
version = "0.1.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"bb8 0.4.0-pre (git+https://github.com/khuey/bb8)",
|
"bb8 0.4.0-pre (git+https://github.com/khuey/bb8.git)",
|
||||||
"bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8)",
|
"bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8.git)",
|
||||||
"furaffinity-rs 0.1.0 (git+https://git.huefox.com/syfaro/furaffinity-rs.git)",
|
"furaffinity-rs 0.1.0 (git+https://git.huefox.com/syfaro/furaffinity-rs.git)",
|
||||||
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
"image 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
"image 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)",
|
||||||
@ -2209,8 +2209,8 @@ dependencies = [
|
|||||||
"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
|
"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
|
||||||
"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
|
"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
|
||||||
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
|
||||||
"checksum bb8 0.4.0-pre (git+https://github.com/khuey/bb8)" = "<none>"
|
"checksum bb8 0.4.0-pre (git+https://github.com/khuey/bb8.git)" = "<none>"
|
||||||
"checksum bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8)" = "<none>"
|
"checksum bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8.git)" = "<none>"
|
||||||
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
|
||||||
"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
|
"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
|
||||||
"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
|
"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"
|
||||||
|
@ -10,7 +10,7 @@ struct NeededPost {
|
|||||||
async fn hash_url(
|
async fn hash_url(
|
||||||
client: std::sync::Arc<reqwest::Client>,
|
client: std::sync::Arc<reqwest::Client>,
|
||||||
url: String,
|
url: String,
|
||||||
) -> (img_hash::ImageHash, i64) {
|
) -> Result<(img_hash::ImageHash, i64), image::ImageError> {
|
||||||
println!("loading {}", url);
|
println!("loading {}", url);
|
||||||
|
|
||||||
let data = client
|
let data = client
|
||||||
@ -23,7 +23,7 @@ async fn hash_url(
|
|||||||
.expect("unable to get bytes");
|
.expect("unable to get bytes");
|
||||||
|
|
||||||
let hasher = furaffinity_rs::get_hasher();
|
let hasher = furaffinity_rs::get_hasher();
|
||||||
let image = image::load_from_memory(&data).expect("unable to parse image");
|
let image = image::load_from_memory(&data)?;
|
||||||
|
|
||||||
let hash = hasher.hash_image(&image);
|
let hash = hasher.hash_image(&image);
|
||||||
let mut bytes: [u8; 8] = [0; 8];
|
let mut bytes: [u8; 8] = [0; 8];
|
||||||
@ -33,7 +33,7 @@ async fn hash_url(
|
|||||||
|
|
||||||
println!("{} - {}", url, num);
|
println!("{} - {}", url, num);
|
||||||
|
|
||||||
(hash, num)
|
Ok((hash, num))
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
@ -94,13 +94,31 @@ async fn main() {
|
|||||||
let client = client.clone();
|
let client = client.clone();
|
||||||
let id = post.id;
|
let id = post.id;
|
||||||
|
|
||||||
hash_url(client, post.full_url.clone()).then(move |(_hash, num)| async move {
|
hash_url(client, post.full_url.clone()).then(move |res| async move {
|
||||||
db.get()
|
match res {
|
||||||
.await
|
Ok((_hash, num)) => {
|
||||||
.unwrap()
|
db.get()
|
||||||
.execute("UPDATE post SET hash = $2 WHERE id = $1", &[&id, &num])
|
.await
|
||||||
.await
|
.unwrap()
|
||||||
.expect("Unable to update hash in database");
|
.execute("UPDATE post SET hash = $2 WHERE id = $1", &[&id, &num])
|
||||||
|
.await
|
||||||
|
.expect("Unable to update hash in database");
|
||||||
|
}
|
||||||
|
Err(e) => {
|
||||||
|
use std::error::Error;
|
||||||
|
let desc = e.description();
|
||||||
|
println!("hashing error - {}", desc);
|
||||||
|
db.get()
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.execute(
|
||||||
|
"UPDATE post SET hash_error = $2 WHERE id = $1",
|
||||||
|
&[&id, &desc],
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("Unable to update hash in database");
|
||||||
|
}
|
||||||
|
};
|
||||||
})
|
})
|
||||||
});
|
});
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user