anthrove/fuzzysearch
4
0
Fork 0
mirror of https://github.com/Syfaro/fuzzysearch.git synced 2024-11-23 15:22:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Make async.

Browse Source
This commit is contained in:
Syfaro 2019-12-18 02:31:10 -06:00
parent ba0aca1c80
commit 97246f502a
3 changed files with 323 additions and 512 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

810
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

5
Cargo.toml
View File

@ -5,8 +5,9 @@ authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018" edition = "2018"
[dependencies] [dependencies]
reqwest = "0.9" reqwest = { git = "https://github.com/seanmonstar/reqwest" }
postgres = { version = "0.17.0-alpha.2", features = ["with-chrono-0_4"] } postgres = { version = "0.17.0-alpha.2", features = ["with-chrono-0_4"] }
tokio = { version = "0.2", features = ["macros", "time"] }
tokio-postgres = { version = "0.5.0-alpha.2" } tokio-postgres = { version = "0.5.0-alpha.2" }
r2d2_postgres = " 0.16.0-alpha.2" r2d2_postgres = " 0.16.0-alpha.2"
r2d2 = "0.8" r2d2 = "0.8"
@ -14,4 +15,4 @@ chrono = "0.4"
[dependencies.furaffinity-rs] [dependencies.furaffinity-rs]
git = "https://git.huefox.com/syfaro/furaffinity-rs" git = "https://git.huefox.com/syfaro/furaffinity-rs"
rev = "35478cb1402b26842900546f5cb1130238866c22" rev = "d6d5a61bfe041e8ecbbe135c43dcbb577961e2f3"

20
src/main.rs
View File

@ -71,11 +71,7 @@ fn insert_submission(
.collect(); .collect();
let hash = sub.hash.clone(); let hash = sub.hash.clone();
let url = sub.content.url();
let url = match &sub.content {
furaffinity_rs::Content::Image(url) => url,
furaffinity_rs::Content::Flash(url) => url,
};
client.execute("INSERT INTO submission (id, artist_id, url, filename, hash, rating, posted_at, description, hash1, hash2, hash3, hash4, hash5, hash6, hash7, hash8) VALUES ($1, $2, $3, $4, decode($5, 'base64'), $6, $7, $8, get_byte(decode($5, 'base64'), 0), get_byte(decode($5, 'base64'), 1), get_byte(decode($5, 'base64'), 2), get_byte(decode($5, 'base64'), 3), get_byte(decode($5, 'base64'), 4), get_byte(decode($5, 'base64'), 5), get_byte(decode($5, 'base64'), 6), get_byte(decode($5, 'base64'), 7))", &[ client.execute("INSERT INTO submission (id, artist_id, url, filename, hash, rating, posted_at, description, hash1, hash2, hash3, hash4, hash5, hash6, hash7, hash8) VALUES ($1, $2, $3, $4, decode($5, 'base64'), $6, $7, $8, get_byte(decode($5, 'base64'), 0), get_byte(decode($5, 'base64'), 1), get_byte(decode($5, 'base64'), 2), get_byte(decode($5, 'base64'), 3), get_byte(decode($5, 'base64'), 4), get_byte(decode($5, 'base64'), 5), get_byte(decode($5, 'base64'), 6), get_byte(decode($5, 'base64'), 7))", &[
&sub.id, &artist_id, &url, &sub.filename, &hash, &sub.rating.serialize(), &sub.posted_at, &sub.description, &sub.id, &artist_id, &url, &sub.filename, &hash, &sub.rating.serialize(), &sub.posted_at, &sub.description,
@ -84,6 +80,7 @@ fn insert_submission(
let stmt = client.prepare( let stmt = client.prepare(
"INSERT INTO tag_to_post (tag_id, post_id) VALUES ($1, $2) ON CONFLICT DO NOTHING", "INSERT INTO tag_to_post (tag_id, post_id) VALUES ($1, $2) ON CONFLICT DO NOTHING",
)?; )?;
for tag_id in tag_ids { for tag_id in tag_ids {
client.execute(&stmt, &[&tag_id, &sub.id])?; client.execute(&stmt, &[&tag_id, &sub.id])?;
} }
@ -95,7 +92,8 @@ fn insert_null_submission(client: &mut Client, id: i32) -> Result<u64, postgres:
client.execute("INSERT INTO SUBMISSION (id) VALUES ($1)", &[&id]) client.execute("INSERT INTO SUBMISSION (id) VALUES ($1)", &[&id])
} }
fn main() { #[tokio::main]
async fn main() {
let (cookie_a, cookie_b) = ( let (cookie_a, cookie_b) = (
std::env::var("FA_A").expect("missing fa cookie a"), std::env::var("FA_A").expect("missing fa cookie a"),
std::env::var("FA_B").expect("missing fa cookie b"), std::env::var("FA_B").expect("missing fa cookie b"),
@ -115,19 +113,19 @@ fn main() {
'main: loop { 'main: loop {
let mut client = pool.get().unwrap(); let mut client = pool.get().unwrap();
let latest_id = fa.latest_id().expect("unable to get latest id"); let latest_id = fa.latest_id().await.expect("unable to get latest id");
for id in ids_to_check(&mut client, latest_id) { for id in ids_to_check(&mut client, latest_id) {
'attempt: for attempt in 0..3 { 'attempt: for attempt in 0..3 {
if !has_submission(&mut client, id) { if !has_submission(&mut client, id) {
println!("loading submission {}", id); println!("loading submission {}", id);
let sub = match fa.get_submission(id) { let sub = match fa.get_submission(id).await {
Ok(sub) => sub, Ok(sub) => sub,
Err(e) => { Err(e) => {
println!("got error: {:?}, retry {}", e.message, e.retry); println!("got error: {:?}, retry {}", e.message, e.retry);
if e.retry { if e.retry {
std::thread::sleep(std::time::Duration::from_secs(attempt + 1)); tokio::time::delay_for(std::time::Duration::from_secs(attempt + 1)).await;
continue 'attempt; continue 'attempt;
} else { } else {
println!("unrecoverable, exiting"); println!("unrecoverable, exiting");
@ -145,7 +143,7 @@ fn main() {
} }
}; };
let sub = match fa.calc_image_hash(sub.clone()) { let sub = match fa.calc_image_hash(sub.clone()).await {
Ok(sub) => sub, Ok(sub) => sub,
Err(e) => { Err(e) => {
println!("unable to hash image: {:?}", e); println!("unable to hash image: {:?}", e);
@ -164,6 +162,6 @@ fn main() {
println!("completed fetch, waiting a minute before loading more"); println!("completed fetch, waiting a minute before loading more");
std::thread::sleep(std::time::Duration::from_secs(60)); tokio::time::delay_for(std::time::Duration::from_secs(60)).await;
} }
} }