Add client.

.gitlab-ci.yml

@@ -11,7 +11,7 @@ build:
   artifacts:
     expire_in: 1 day
     paths:
-      - ./bkapi
+      - ./bkapi/bkapi
   cache:
     - key:
         files:
@@ -20,8 +20,8 @@ build:
         - target/
         - .cargo/
   script:
-    - cargo build --release --verbose
+    - cargo build --bin bkapi --release --verbose
-    - mv ./target/release/bkapi ./bkapi
+    - mv ./target/release/bkapi ./bkapi/bkapi
 
 docker:
   image:

Cargo.lock

@@ -337,6 +337,21 @@ dependencies = [
  "tracing-unwrap",
 ]
 
+[[package]]
+name = "bkapi-client"
+version = "0.1.0"
+dependencies = [
+ "futures",
+ "opentelemetry",
+ "opentelemetry-http",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "tokio",
+ "tracing",
+ "tracing-opentelemetry",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.9.0"
@@ -449,6 +464,22 @@ dependencies = [
  "version_check",
 ]
 
+[[package]]
+name = "core-foundation"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
+
 [[package]]
 name = "cpufeatures"
 version = "0.1.5"
@@ -623,6 +654,21 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.0.1"
@@ -842,12 +888,66 @@ dependencies = [
  "itoa",
 ]
 
+[[package]]
+name = "http-body"
+version = "0.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
 [[package]]
 name = "httparse"
 version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"
 
+[[package]]
+name = "httpdate"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440"
+
+[[package]]
+name = "hyper"
+version = "0.14.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
 [[package]]
 name = "idna"
 version = "0.2.3"
@@ -884,6 +984,12 @@ version = "1.1.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f"
 
+[[package]]
+name = "ipnet"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
+
 [[package]]
 name = "itoa"
 version = "0.4.7"
@@ -1051,6 +1157,24 @@ dependencies = [
  "winapi",
 ]
 
+[[package]]
+name = "native-tls"
+version = "0.2.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4"
+dependencies = [
+ "lazy_static",
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
 [[package]]
 name = "nom"
 version = "6.1.2"
@@ -1114,6 +1238,39 @@ version = "0.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
 
+[[package]]
+name = "openssl"
+version = "0.10.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885"
+dependencies = [
+ "bitflags",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.65"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
+dependencies = [
+ "autocfg",
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
 [[package]]
 name = "opentelemetry"
 version = "0.15.0"
@@ -1133,6 +1290,19 @@ dependencies = [
  "tokio-stream",
 ]
 
+[[package]]
+name = "opentelemetry-http"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1eeb0d4b230c69aa425f9c3aadf02ce7aa4b2bc9a4e9fbe59614108232078263"
+dependencies = [
+ "async-trait",
+ "bytes",
+ "futures-util",
+ "http",
+ "opentelemetry",
+]
+
 [[package]]
 name = "opentelemetry-jaeger"
 version = "0.14.0"
@@ -1233,6 +1403,12 @@ version = "0.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
+[[package]]
+name = "pkg-config"
+version = "0.3.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
+
 [[package]]
 name = "ppv-lite86"
 version = "0.2.10"
@@ -1397,6 +1573,50 @@ version = "0.6.25"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
 
+[[package]]
+name = "remove_dir_all"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "reqwest"
+version = "0.11.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22"
+dependencies = [
+ "base64",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "lazy_static",
+ "log",
+ "mime",
+ "native-tls",
+ "percent-encoding",
+ "pin-project-lite",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "tokio",
+ "tokio-native-tls",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+ "winreg",
+]
+
 [[package]]
 name = "ring"
 version = "0.16.20"
@@ -1449,6 +1669,16 @@ version = "1.0.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
 
+[[package]]
+name = "schannel"
+version = "0.1.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
+dependencies = [
+ "lazy_static",
+ "winapi",
+]
+
 [[package]]
 name = "scopeguard"
 version = "1.1.0"
@@ -1465,6 +1695,29 @@ dependencies = [
  "untrusted",
 ]
 
+[[package]]
+name = "security-framework"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
+dependencies = [
+ "bitflags",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
 [[package]]
 name = "semver"
 version = "0.9.0"
@@ -1821,6 +2074,20 @@ version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
 
+[[package]]
+name = "tempfile"
+version = "3.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "rand",
+ "redox_syscall",
+ "remove_dir_all",
+ "winapi",
+]
+
 [[package]]
 name = "thiserror"
 version = "1.0.26"
@@ -1941,9 +2208,31 @@ dependencies = [
  "parking_lot",
  "pin-project-lite",
  "signal-hook-registry",
+ "tokio-macros",
  "winapi",
 ]
 
+[[package]]
+name = "tokio-macros"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
 [[package]]
 name = "tokio-rustls"
 version = "0.22.0"
@@ -1980,6 +2269,12 @@ dependencies = [
  "tokio",
 ]
 
+[[package]]
+name = "tower-service"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
+
 [[package]]
 name = "tracing"
 version = "0.1.26"
@@ -2108,6 +2403,12 @@ version = "0.3.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "622b09ce2fe2df4618636fb92176d205662f59803f39e70d1c333393082de96c"
 
+[[package]]
+name = "try-lock"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
+
 [[package]]
 name = "typenum"
 version = "1.13.0"
@@ -2183,12 +2484,28 @@ dependencies = [
  "getrandom",
 ]
 
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
 [[package]]
 name = "version_check"
 version = "0.9.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
 
+[[package]]
+name = "want"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
+dependencies = [
+ "log",
+ "try-lock",
+]
+
 [[package]]
 name = "wasi"
 version = "0.10.2+wasi-snapshot-preview1"
@@ -2202,6 +2519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
 dependencies = [
  "cfg-if",
+ "serde",
+ "serde_json",
  "wasm-bindgen-macro",
 ]
 
@@ -2220,6 +2539,18 @@ dependencies = [
  "wasm-bindgen-shared",
 ]
 
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
 [[package]]
 name = "wasm-bindgen-macro"
 version = "0.2.74"
@@ -2310,6 +2641,15 @@ version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
 
+[[package]]
+name = "winreg"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
+dependencies = [
+ "winapi",
+]
+
 [[package]]
 name = "wyz"
 version = "0.2.0"

Cargo.toml

@@ -1,38 +1,5 @@
-[package]
+[workspace]
-name = "bkapi"
+members = [
-version = "0.1.0"
+    "bkapi",
-authors = ["Syfaro <syfaro@huefox.com>"]
+    "bkapi-client",
-edition = "2018"
+]
-
-[dependencies]
-envconfig = "0.10"
-thiserror = "1"
-
-tracing = "0.1"
-tracing-subscriber = "0.2"
-tracing-unwrap = "0.9"
-tracing-opentelemetry = "0.14"
-
-opentelemetry = { version = "0.15", features = ["rt-tokio"] }
-opentelemetry-jaeger = "0.14"
-
-lazy_static = "1"
-prometheus = { version = "0.12", features = ["process"] }
-
-bk-tree = "0.4.0"
-hamming = "0.1"
-
-futures = "0.3"
-tokio = { version = "1", features = ["sync"] }
-
-serde = { version = "1", features = ["derive"] }
-serde_json = "1"
-
-actix-web = "4.0.0-beta.8"
-actix-http = "3.0.0-beta.8"
-actix-service = "2"
-tracing-actix-web = { version = "0.4.0-beta.9", features = ["opentelemetry_0_15"] }
-
-[dependencies.sqlx]
-version = "0.5"
-features = ["runtime-actix-rustls", "postgres"]

bkapi-client/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "bkapi-client"
+version = "0.1.0"
+authors = ["Syfaro <syfaro@huefox.com>"]
+edition = "2018"
+
+[dependencies]
+tracing = "0.1"
+tracing-opentelemetry = "0.14"
+
+opentelemetry = "0.15"
+opentelemetry-http = "0.4"
+
+futures = "0.3"
+
+reqwest = { version = "0.11", features = ["json"] }
+
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+[dev-dependencies]
+tokio = { version = "1", features = ["full"] }

bkapi-client/src/lib.rs

@@ -0,0 +1,129 @@
+use futures::TryStreamExt;
+use serde::{Deserialize, Serialize};
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SearchResults {
+    pub hash: i64,
+    pub distance: u64,
+
+    pub hashes: Vec<SearchResult>,
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct SearchResult {
+    pub hash: i64,
+    pub distance: u64,
+}
+
+pub struct BKApiClient {
+    pub endpoint: String,
+    client: reqwest::Client,
+}
+
+impl BKApiClient {
+    pub fn new<E>(endpoint: E) -> Self
+    where
+        E: Into<String>,
+    {
+        Self {
+            endpoint: endpoint.into(),
+            client: reqwest::Client::default(),
+        }
+    }
+
+    #[tracing::instrument(err, skip(self))]
+    pub async fn search(&self, hash: i64, distance: u64) -> Result<SearchResults, reqwest::Error> {
+        let results = self
+            .client
+            .get(&self.endpoint)
+            .query(&[
+                ("hash", hash.to_string()),
+                ("distance", distance.to_string()),
+            ])
+            .inject_context()
+            .send()
+            .await?
+            .error_for_status()?
+            .json()
+            .await?;
+
+        Ok(results)
+    }
+
+    #[tracing::instrument(err, skip(self))]
+    pub async fn search_many(
+        &self,
+        hashes: &[i64],
+        distance: u64,
+    ) -> Result<Vec<SearchResults>, reqwest::Error> {
+        let mut futs = futures::stream::FuturesOrdered::new();
+        for hash in hashes {
+            futs.push(self.search(*hash, distance));
+        }
+
+        futs.try_collect().await
+    }
+}
+
+trait InjectContext {
+    fn inject_context(self) -> Self;
+}
+
+impl InjectContext for reqwest::RequestBuilder {
+    fn inject_context(self: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
+        use tracing_opentelemetry::OpenTelemetrySpanExt;
+
+        let mut headers: reqwest::header::HeaderMap = Default::default();
+
+        let cx = tracing::Span::current().context();
+        opentelemetry::global::get_text_map_propagator(|propagator| {
+            propagator.inject_context(&cx, &mut opentelemetry_http::HeaderInjector(&mut headers))
+        });
+
+        self.headers(headers)
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    fn get_test_endpoint() -> String {
+        std::env::var("BKAPI_ENDPOINT").expect("tests require BKAPI_ENDPOINT to be set")
+    }
+
+    #[tokio::test]
+    async fn test_search() {
+        let endpoint = get_test_endpoint();
+        let client = super::BKApiClient::new(endpoint);
+
+        let results = client.search(0, 3).await;
+        println!("{:?}", results);
+        let results = results.expect("search should not have error");
+        assert_eq!(
+            0, results.hash,
+            "returned hash should have been {} but was {}",
+            0, results.hash
+        );
+        assert_eq!(
+            3, results.distance,
+            "returned distance should have been {} but was {}",
+            3, results.distance
+        );
+    }
+
+    #[tokio::test]
+    async fn test_search_many() {
+        let endpoint = get_test_endpoint();
+        let client = super::BKApiClient::new(endpoint);
+
+        let results = client.search_many(&[0, 1_000_000], 7).await;
+        println!("{:?}", results);
+        let results = results.expect("search should not have error");
+        assert_eq!(
+            2,
+            results.len(),
+            "returned results were missing search, expected {} items but found {} items",
+            2,
+            results.len()
+        );
+    }
+}

bkapi/Cargo.toml

@@ -0,0 +1,38 @@
+[package]
+name = "bkapi"
+version = "0.1.0"
+authors = ["Syfaro <syfaro@huefox.com>"]
+edition = "2018"
+
+[dependencies]
+envconfig = "0.10"
+thiserror = "1"
+
+tracing = "0.1"
+tracing-subscriber = "0.2"
+tracing-unwrap = "0.9"
+tracing-opentelemetry = "0.14"
+
+opentelemetry = { version = "0.15", features = ["rt-tokio"] }
+opentelemetry-jaeger = "0.14"
+
+lazy_static = "1"
+prometheus = { version = "0.12", features = ["process"] }
+
+bk-tree = "0.4.0"
+hamming = "0.1"
+
+futures = "0.3"
+tokio = { version = "1", features = ["sync"] }
+
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+
+actix-web = "4.0.0-beta.8"
+actix-http = "3.0.0-beta.8"
+actix-service = "2"
+tracing-actix-web = { version = "0.4.0-beta.9", features = ["opentelemetry_0_15"] }
+
+[dependencies.sqlx]
+version = "0.5"
+features = ["runtime-actix-rustls", "postgres"]

bkapi/Dockerfile

@@ -1,4 +1,4 @@
 FROM debian:buster-slim
 EXPOSE 3000
-COPY ./bkapi /bin/bkapi
+COPY ./bkapi/bkapi /bin/bkapi
 CMD ["/bin/bkapi"]