anthrove/fuzzysearch
4
0
Fork 0
mirror of https://github.com/Syfaro/fuzzysearch.git synced 2024-09-20 13:59:27 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tolerate and log hashing errors.

Browse Source
This commit is contained in:
Syfaro 2020-01-11 23:56:37 -06:00
parent 797fa4cb11
commit f02a8db8c0
2 changed files with 35 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
Cargo.lock generated
View File

@ -59,7 +59,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
[[package]] [[package]]
name = "bb8" name = "bb8"
version = "0.4.0-pre" version = "0.4.0-pre"
source = "git+https://github.com/khuey/bb8#787b525621a27209da4fe5d0025c7e47a0ff9161" source = "git+https://github.com/khuey/bb8.git#787b525621a27209da4fe5d0025c7e47a0ff9161"
dependencies = [ dependencies = [
"async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -69,10 +69,10 @@ dependencies = [
[[package]] [[package]]
name = "bb8-postgres" name = "bb8-postgres"
version = "0.4.0-pre" version = "0.4.0-pre"
source = "git+https://github.com/khuey/bb8#787b525621a27209da4fe5d0025c7e47a0ff9161" source = "git+https://github.com/khuey/bb8.git#787b525621a27209da4fe5d0025c7e47a0ff9161"
dependencies = [ dependencies = [
"async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)", "async-trait 0.1.22 (registry+https://github.com/rust-lang/crates.io-index)",
"bb8 0.4.0-pre (git+https://github.com/khuey/bb8)", "bb8 0.4.0-pre (git+https://github.com/khuey/bb8.git)",
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)", "tokio 0.2.9 (registry+https://github.com/rust-lang/crates.io-index)",
"tokio-postgres 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)", "tokio-postgres 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)",
@ -299,8 +299,8 @@ dependencies = [
name = "e621-watcher" name = "e621-watcher"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"bb8 0.4.0-pre (git+https://github.com/khuey/bb8)", "bb8 0.4.0-pre (git+https://github.com/khuey/bb8.git)",
"bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8)", "bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8.git)",
"furaffinity-rs 0.1.0 (git+https://git.huefox.com/syfaro/furaffinity-rs.git)", "furaffinity-rs 0.1.0 (git+https://git.huefox.com/syfaro/furaffinity-rs.git)",
"futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", "futures 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)",
"image 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)", "image 0.22.3 (registry+https://github.com/rust-lang/crates.io-index)",
@ -2209,8 +2209,8 @@ dependencies = [
"checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" "checksum autocfg 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d"
"checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" "checksum base64 0.10.1 (registry+https://github.com/rust-lang/crates.io-index)" = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e"
"checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
"checksum bb8 0.4.0-pre (git+https://github.com/khuey/bb8)" = "<none>" "checksum bb8 0.4.0-pre (git+https://github.com/khuey/bb8.git)" = "<none>"
"checksum bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8)" = "<none>" "checksum bb8-postgres 0.4.0-pre (git+https://github.com/khuey/bb8.git)" = "<none>"
"checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
"checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b" "checksum block-buffer 0.7.3 (registry+https://github.com/rust-lang/crates.io-index)" = "c0940dc441f31689269e10ac70eb1002a3a1d3ad1390e030043662eb7fe4688b"
"checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5" "checksum block-padding 0.1.5 (registry+https://github.com/rust-lang/crates.io-index)" = "fa79dedbb091f449f1f39e53edf88d5dbe95f895dae6135a8d7b881fb5af73f5"

26
src/bin/load_hashes.rs
View File

@ -10,7 +10,7 @@ struct NeededPost {
async fn hash_url( async fn hash_url(
client: std::sync::Arc<reqwest::Client>, client: std::sync::Arc<reqwest::Client>,
url: String, url: String,
) -> (img_hash::ImageHash, i64) { ) -> Result<(img_hash::ImageHash, i64), image::ImageError> {
println!("loading {}", url); println!("loading {}", url);
let data = client let data = client
@ -23,7 +23,7 @@ async fn hash_url(
.expect("unable to get bytes"); .expect("unable to get bytes");
let hasher = furaffinity_rs::get_hasher(); let hasher = furaffinity_rs::get_hasher();
let image = image::load_from_memory(&data).expect("unable to parse image"); let image = image::load_from_memory(&data)?;
let hash = hasher.hash_image(&image); let hash = hasher.hash_image(&image);
let mut bytes: [u8; 8] = [0; 8]; let mut bytes: [u8; 8] = [0; 8];
@ -33,7 +33,7 @@ async fn hash_url(
println!("{} - {}", url, num); println!("{} - {}", url, num);
(hash, num) Ok((hash, num))
} }
#[tokio::main] #[tokio::main]
@ -94,13 +94,31 @@ async fn main() {
let client = client.clone(); let client = client.clone();
let id = post.id; let id = post.id;
hash_url(client, post.full_url.clone()).then(move |(_hash, num)| async move { hash_url(client, post.full_url.clone()).then(move |res| async move {
match res {
Ok((_hash, num)) => {
db.get() db.get()
.await .await
.unwrap() .unwrap()
.execute("UPDATE post SET hash = $2 WHERE id = $1", &[&id, &num]) .execute("UPDATE post SET hash = $2 WHERE id = $1", &[&id, &num])
.await .await
.expect("Unable to update hash in database"); .expect("Unable to update hash in database");
}
Err(e) => {
use std::error::Error;
let desc = e.description();
println!("hashing error - {}", desc);
db.get()
.await
.unwrap()
.execute(
"UPDATE post SET hash_error = $2 WHERE id = $1",
&[&id, &desc],
)
.await
.expect("Unable to update hash in database");
}
};
}) })
}); });