anthrove/fuzzysearch
4
0
Fork 0
mirror of https://github.com/Syfaro/fuzzysearch.git synced 2024-11-23 15:22:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Load next posts to get hashes for in background.

Browse Source
This commit is contained in:
Syfaro 2020-01-13 16:46:09 -06:00
parent af15b20f8e
commit 5dc771421c
1 changed files with 45 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

76
src/bin/load_hashes.rs
View File

@ -34,34 +34,10 @@ async fn hash_url(
Ok((hash, num))
}
#[tokio::main]
async fn main() {
let dsn = std::env::var("POSTGRES_DSN").expect("missing postgres dsn");
use std::str::FromStr;
let manager = PostgresConnectionManager::new(
tokio_postgres::Config::from_str(&dsn).expect("unable to parse postgres dsn"),
tokio_postgres::NoTls,
);
let pool = Pool::builder()
.build(manager)
.await
.expect("unable to build pool");
let client = reqwest::Client::builder()
.user_agent("Syfaro test client syfaro@huefox.com")
.build()
.expect("Unable to build http client");
let client = std::sync::Arc::new(client);
loop {
println!("getting next 384 posts");
let db = pool.clone();
let needed_posts: Vec<_> = db
.get()
async fn load_next_posts(
db: Pool<PostgresConnectionManager<tokio_postgres::NoTls>>,
) -> Vec<NeededPost> {
db.get()
.await
.unwrap()
.query(
@ -86,11 +62,47 @@ async fn main() {
id: row.get("id"),
full_url: row.get("file_url"),
})
.collect();
.collect()
}
#[tokio::main]
async fn main() {
let dsn = std::env::var("POSTGRES_DSN").expect("missing postgres dsn");
use std::str::FromStr;
let manager = PostgresConnectionManager::new(
tokio_postgres::Config::from_str(&dsn).expect("unable to parse postgres dsn"),
tokio_postgres::NoTls,
);
let pool = Pool::builder()
.build(manager)
.await
.expect("unable to build pool");
let client = reqwest::Client::builder()
.user_agent("Syfaro test client syfaro@huefox.com")
.build()
.expect("Unable to build http client");
let client = std::sync::Arc::new(client);
let mut needed_posts = load_next_posts(pool.clone()).await;
loop {
println!("running loop");
if needed_posts.is_empty() {
println!("no posts, waiting a minute");
tokio::time::delay_for(std::time::Duration::from_secs(60)).await;
continue;
}
let db = pool.clone();
let posts_fut = tokio::spawn(async move { load_next_posts(db).await });
for chunk in needed_posts.chunks(8) {
let futs = chunk.iter().map(|post| {
let db = db.clone();
let db = pool.clone();
let client = client.clone();
let id = post.id;
@ -116,7 +128,7 @@ async fn main() {
&[&id, &desc],
)
.await
.expect("Unable to update hash in database");
.expect("Unable to update hash error in database");
}
};
})
@ -124,5 +136,7 @@ async fn main() {
futures::future::join_all(futs).await;
}
needed_posts = posts_fut.await.unwrap();
}
}