From f040cf3ddedc3545b4b58387cb237122d8873608 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Thu, 22 Apr 2021 22:33:18 -0400 Subject: [PATCH] Revert using UNION instead of UNION ALL. --- fuzzysearch/sqlx-data.json | 4 ++-- fuzzysearch/src/main.rs | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/fuzzysearch/sqlx-data.json b/fuzzysearch/sqlx-data.json index de91322..b4a6cdb 100644 --- a/fuzzysearch/sqlx-data.json +++ b/fuzzysearch/sqlx-data.json @@ -173,8 +173,8 @@ ] } }, - "7953d382d35d07effb803c1b80c6efba81e3db90d0abfe3a40cd66b9106ce44d": { - "query": "SELECT hash_int hash FROM submission WHERE hash_int IS NOT NULL\n UNION\n SELECT hash FROM e621 WHERE hash IS NOT NULL\n UNION\n SELECT hash FROM tweet_media WHERE hash IS NOT NULL\n UNION\n SELECT hash FROM weasyl WHERE hash IS NOT NULL", + "d6a5ed9266d7cf6f08df5e0160c59843a15ef5ef3ccac2e5ccfe2251a50bd0dc": { + "query": "SELECT hash_int hash FROM submission WHERE hash_int IS NOT NULL\n UNION ALL\n SELECT hash FROM e621 WHERE hash IS NOT NULL\n UNION ALL\n SELECT hash FROM tweet_media WHERE hash IS NOT NULL\n UNION ALL\n SELECT hash FROM weasyl WHERE hash IS NOT NULL", "describe": { "columns": [ { diff --git a/fuzzysearch/src/main.rs b/fuzzysearch/src/main.rs index c762c42..f6fba39 100644 --- a/fuzzysearch/src/main.rs +++ b/fuzzysearch/src/main.rs @@ -172,11 +172,11 @@ async fn create_tree(conn: &Pool) -> bk_tree::BKTree { let mut rows = sqlx::query!( "SELECT hash_int hash FROM submission WHERE hash_int IS NOT NULL - UNION + UNION ALL SELECT hash FROM e621 WHERE hash IS NOT NULL - UNION + UNION ALL SELECT hash FROM tweet_media WHERE hash IS NOT NULL - UNION + UNION ALL SELECT hash FROM weasyl WHERE hash IS NOT NULL" ) .fetch(conn); @@ -185,9 +185,12 @@ async fn create_tree(conn: &Pool) -> bk_tree::BKTree { while let Some(row) = rows.try_next().await.expect("Unable to get row") { if let Some(hash) = row.hash { - tree.add(Node::new(hash)); - count += 1; + let node = Node::new(hash); + if tree.find_exact(&node).is_none() { + tree.add(node); + } + count += 1; if count % 250_000 == 0 { tracing::debug!(count, "Made progress in loading tree rows"); }