Load next posts to get hashes for in background.

src/bin/load_hashes.rs

@@ -34,34 +34,10 @@ async fn hash_url(
     Ok((hash, num))
 }
 
-#[tokio::main]
+async fn load_next_posts(
-async fn main() {
+    db: Pool<PostgresConnectionManager<tokio_postgres::NoTls>>,
-    let dsn = std::env::var("POSTGRES_DSN").expect("missing postgres dsn");
+) -> Vec<NeededPost> {
-
+    db.get()
-    use std::str::FromStr;
-    let manager = PostgresConnectionManager::new(
-        tokio_postgres::Config::from_str(&dsn).expect("unable to parse postgres dsn"),
-        tokio_postgres::NoTls,
-    );
-
-    let pool = Pool::builder()
-        .build(manager)
-        .await
-        .expect("unable to build pool");
-
-    let client = reqwest::Client::builder()
-        .user_agent("Syfaro test client syfaro@huefox.com")
-        .build()
-        .expect("Unable to build http client");
-    let client = std::sync::Arc::new(client);
-
-    loop {
-        println!("getting next 384 posts");
-
-        let db = pool.clone();
-
-        let needed_posts: Vec<_> = db
-            .get()
         .await
         .unwrap()
         .query(
@@ -86,11 +62,47 @@ async fn main() {
             id: row.get("id"),
             full_url: row.get("file_url"),
         })
-            .collect();
+        .collect()
+}
+
+#[tokio::main]
+async fn main() {
+    let dsn = std::env::var("POSTGRES_DSN").expect("missing postgres dsn");
+
+    use std::str::FromStr;
+    let manager = PostgresConnectionManager::new(
+        tokio_postgres::Config::from_str(&dsn).expect("unable to parse postgres dsn"),
+        tokio_postgres::NoTls,
+    );
+
+    let pool = Pool::builder()
+        .build(manager)
+        .await
+        .expect("unable to build pool");
+
+    let client = reqwest::Client::builder()
+        .user_agent("Syfaro test client syfaro@huefox.com")
+        .build()
+        .expect("Unable to build http client");
+    let client = std::sync::Arc::new(client);
+
+    let mut needed_posts = load_next_posts(pool.clone()).await;
+
+    loop {
+        println!("running loop");
+
+        if needed_posts.is_empty() {
+            println!("no posts, waiting a minute");
+            tokio::time::delay_for(std::time::Duration::from_secs(60)).await;
+            continue;
+        }
+
+        let db = pool.clone();
+        let posts_fut = tokio::spawn(async move { load_next_posts(db).await });
 
         for chunk in needed_posts.chunks(8) {
             let futs = chunk.iter().map(|post| {
-                let db = db.clone();
+                let db = pool.clone();
                 let client = client.clone();
                 let id = post.id;
 
@@ -116,7 +128,7 @@ async fn main() {
                                     &[&id, &desc],
                                 )
                                 .await
-                                .expect("Unable to update hash in database");
+                                .expect("Unable to update hash error in database");
                         }
                     };
                 })
@@ -124,5 +136,7 @@ async fn main() {
 
             futures::future::join_all(futs).await;
         }
+
+        needed_posts = posts_fut.await.unwrap();
     }
 }