From 2ef64e6d7008e6952c11c7ca386d0eacc4001c99 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Wed, 28 Jul 2021 15:43:13 -0400 Subject: [PATCH] Add client. --- .gitlab-ci.yml | 6 +- Cargo.lock | 340 +++++++++++++++++++++++++++++++++ Cargo.toml | 43 +---- bkapi-client/Cargo.toml | 22 +++ bkapi-client/src/lib.rs | 129 +++++++++++++ bkapi/Cargo.toml | 38 ++++ Dockerfile => bkapi/Dockerfile | 2 +- {src => bkapi/src}/main.rs | 0 {src => bkapi/src}/tree.rs | 0 9 files changed, 538 insertions(+), 42 deletions(-) create mode 100644 bkapi-client/Cargo.toml create mode 100644 bkapi-client/src/lib.rs create mode 100644 bkapi/Cargo.toml rename Dockerfile => bkapi/Dockerfile (64%) rename {src => bkapi/src}/main.rs (100%) rename {src => bkapi/src}/tree.rs (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 936a05e..49c8d71 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -11,7 +11,7 @@ build: artifacts: expire_in: 1 day paths: - - ./bkapi + - ./bkapi/bkapi cache: - key: files: @@ -20,8 +20,8 @@ build: - target/ - .cargo/ script: - - cargo build --release --verbose - - mv ./target/release/bkapi ./bkapi + - cargo build --bin bkapi --release --verbose + - mv ./target/release/bkapi ./bkapi/bkapi docker: image: diff --git a/Cargo.lock b/Cargo.lock index 3659eeb..b39fa97 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -337,6 +337,21 @@ dependencies = [ "tracing-unwrap", ] +[[package]] +name = "bkapi-client" +version = "0.1.0" +dependencies = [ + "futures", + "opentelemetry", + "opentelemetry-http", + "reqwest", + "serde", + "serde_json", + "tokio", + "tracing", + "tracing-opentelemetry", +] + [[package]] name = "block-buffer" version = "0.9.0" @@ -449,6 +464,22 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cpufeatures" version = "0.1.5" @@ -623,6 +654,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.1" @@ -842,12 +888,66 @@ dependencies = [ "itoa", ] +[[package]] +name = "http-body" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + [[package]] name = "httparse" version = "1.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" +[[package]] +name = "httpdate" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" + +[[package]] +name = "hyper" +version = "0.14.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.2.3" @@ -884,6 +984,12 @@ version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" +[[package]] +name = "ipnet" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" + [[package]] name = "itoa" version = "0.4.7" @@ -1051,6 +1157,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "native-tls" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "nom" version = "6.1.2" @@ -1114,6 +1238,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "once_cell", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" + +[[package]] +name = "openssl-sys" +version = "0.9.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.15.0" @@ -1133,6 +1290,19 @@ dependencies = [ "tokio-stream", ] +[[package]] +name = "opentelemetry-http" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1eeb0d4b230c69aa425f9c3aadf02ce7aa4b2bc9a4e9fbe59614108232078263" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "opentelemetry", +] + [[package]] name = "opentelemetry-jaeger" version = "0.14.0" @@ -1233,6 +1403,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1397,6 +1573,50 @@ version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" +[[package]] +name = "remove_dir_all" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" +dependencies = [ + "winapi", +] + +[[package]] +name = "reqwest" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" +dependencies = [ + "base64", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_json", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", +] + [[package]] name = "ring" version = "0.16.20" @@ -1449,6 +1669,16 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -1465,6 +1695,29 @@ dependencies = [ "untrusted", ] +[[package]] +name = "security-framework" +version = "2.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "0.9.0" @@ -1821,6 +2074,20 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" +[[package]] +name = "tempfile" +version = "3.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" +dependencies = [ + "cfg-if", + "libc", + "rand", + "redox_syscall", + "remove_dir_all", + "winapi", +] + [[package]] name = "thiserror" version = "1.0.26" @@ -1941,9 +2208,31 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", + "tokio-macros", "winapi", ] +[[package]] +name = "tokio-macros" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "tokio-native-tls" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.22.0" @@ -1980,6 +2269,12 @@ dependencies = [ "tokio", ] +[[package]] +name = "tower-service" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" + [[package]] name = "tracing" version = "0.1.26" @@ -2108,6 +2403,12 @@ version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "622b09ce2fe2df4618636fb92176d205662f59803f39e70d1c333393082de96c" +[[package]] +name = "try-lock" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" + [[package]] name = "typenum" version = "1.13.0" @@ -2183,12 +2484,28 @@ dependencies = [ "getrandom", ] +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" + [[package]] name = "version_check" version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe" +[[package]] +name = "want" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0" +dependencies = [ + "log", + "try-lock", +] + [[package]] name = "wasi" version = "0.10.2+wasi-snapshot-preview1" @@ -2202,6 +2519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd" dependencies = [ "cfg-if", + "serde", + "serde_json", "wasm-bindgen-macro", ] @@ -2220,6 +2539,18 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" +dependencies = [ + "cfg-if", + "js-sys", + "wasm-bindgen", + "web-sys", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.74" @@ -2310,6 +2641,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi", +] + [[package]] name = "wyz" version = "0.2.0" diff --git a/Cargo.toml b/Cargo.toml index f6c6598..03f2ae1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,38 +1,5 @@ -[package] -name = "bkapi" -version = "0.1.0" -authors = ["Syfaro "] -edition = "2018" - -[dependencies] -envconfig = "0.10" -thiserror = "1" - -tracing = "0.1" -tracing-subscriber = "0.2" -tracing-unwrap = "0.9" -tracing-opentelemetry = "0.14" - -opentelemetry = { version = "0.15", features = ["rt-tokio"] } -opentelemetry-jaeger = "0.14" - -lazy_static = "1" -prometheus = { version = "0.12", features = ["process"] } - -bk-tree = "0.4.0" -hamming = "0.1" - -futures = "0.3" -tokio = { version = "1", features = ["sync"] } - -serde = { version = "1", features = ["derive"] } -serde_json = "1" - -actix-web = "4.0.0-beta.8" -actix-http = "3.0.0-beta.8" -actix-service = "2" -tracing-actix-web = { version = "0.4.0-beta.9", features = ["opentelemetry_0_15"] } - -[dependencies.sqlx] -version = "0.5" -features = ["runtime-actix-rustls", "postgres"] +[workspace] +members = [ + "bkapi", + "bkapi-client", +] diff --git a/bkapi-client/Cargo.toml b/bkapi-client/Cargo.toml new file mode 100644 index 0000000..6926467 --- /dev/null +++ b/bkapi-client/Cargo.toml @@ -0,0 +1,22 @@ +[package] +name = "bkapi-client" +version = "0.1.0" +authors = ["Syfaro "] +edition = "2018" + +[dependencies] +tracing = "0.1" +tracing-opentelemetry = "0.14" + +opentelemetry = "0.15" +opentelemetry-http = "0.4" + +futures = "0.3" + +reqwest = { version = "0.11", features = ["json"] } + +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[dev-dependencies] +tokio = { version = "1", features = ["full"] } diff --git a/bkapi-client/src/lib.rs b/bkapi-client/src/lib.rs new file mode 100644 index 0000000..472f8d6 --- /dev/null +++ b/bkapi-client/src/lib.rs @@ -0,0 +1,129 @@ +use futures::TryStreamExt; +use serde::{Deserialize, Serialize}; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SearchResults { + pub hash: i64, + pub distance: u64, + + pub hashes: Vec, +} + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct SearchResult { + pub hash: i64, + pub distance: u64, +} + +pub struct BKApiClient { + pub endpoint: String, + client: reqwest::Client, +} + +impl BKApiClient { + pub fn new(endpoint: E) -> Self + where + E: Into, + { + Self { + endpoint: endpoint.into(), + client: reqwest::Client::default(), + } + } + + #[tracing::instrument(err, skip(self))] + pub async fn search(&self, hash: i64, distance: u64) -> Result { + let results = self + .client + .get(&self.endpoint) + .query(&[ + ("hash", hash.to_string()), + ("distance", distance.to_string()), + ]) + .inject_context() + .send() + .await? + .error_for_status()? + .json() + .await?; + + Ok(results) + } + + #[tracing::instrument(err, skip(self))] + pub async fn search_many( + &self, + hashes: &[i64], + distance: u64, + ) -> Result, reqwest::Error> { + let mut futs = futures::stream::FuturesOrdered::new(); + for hash in hashes { + futs.push(self.search(*hash, distance)); + } + + futs.try_collect().await + } +} + +trait InjectContext { + fn inject_context(self) -> Self; +} + +impl InjectContext for reqwest::RequestBuilder { + fn inject_context(self: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + use tracing_opentelemetry::OpenTelemetrySpanExt; + + let mut headers: reqwest::header::HeaderMap = Default::default(); + + let cx = tracing::Span::current().context(); + opentelemetry::global::get_text_map_propagator(|propagator| { + propagator.inject_context(&cx, &mut opentelemetry_http::HeaderInjector(&mut headers)) + }); + + self.headers(headers) + } +} + +#[cfg(test)] +mod tests { + fn get_test_endpoint() -> String { + std::env::var("BKAPI_ENDPOINT").expect("tests require BKAPI_ENDPOINT to be set") + } + + #[tokio::test] + async fn test_search() { + let endpoint = get_test_endpoint(); + let client = super::BKApiClient::new(endpoint); + + let results = client.search(0, 3).await; + println!("{:?}", results); + let results = results.expect("search should not have error"); + assert_eq!( + 0, results.hash, + "returned hash should have been {} but was {}", + 0, results.hash + ); + assert_eq!( + 3, results.distance, + "returned distance should have been {} but was {}", + 3, results.distance + ); + } + + #[tokio::test] + async fn test_search_many() { + let endpoint = get_test_endpoint(); + let client = super::BKApiClient::new(endpoint); + + let results = client.search_many(&[0, 1_000_000], 7).await; + println!("{:?}", results); + let results = results.expect("search should not have error"); + assert_eq!( + 2, + results.len(), + "returned results were missing search, expected {} items but found {} items", + 2, + results.len() + ); + } +} diff --git a/bkapi/Cargo.toml b/bkapi/Cargo.toml new file mode 100644 index 0000000..f6c6598 --- /dev/null +++ b/bkapi/Cargo.toml @@ -0,0 +1,38 @@ +[package] +name = "bkapi" +version = "0.1.0" +authors = ["Syfaro "] +edition = "2018" + +[dependencies] +envconfig = "0.10" +thiserror = "1" + +tracing = "0.1" +tracing-subscriber = "0.2" +tracing-unwrap = "0.9" +tracing-opentelemetry = "0.14" + +opentelemetry = { version = "0.15", features = ["rt-tokio"] } +opentelemetry-jaeger = "0.14" + +lazy_static = "1" +prometheus = { version = "0.12", features = ["process"] } + +bk-tree = "0.4.0" +hamming = "0.1" + +futures = "0.3" +tokio = { version = "1", features = ["sync"] } + +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +actix-web = "4.0.0-beta.8" +actix-http = "3.0.0-beta.8" +actix-service = "2" +tracing-actix-web = { version = "0.4.0-beta.9", features = ["opentelemetry_0_15"] } + +[dependencies.sqlx] +version = "0.5" +features = ["runtime-actix-rustls", "postgres"] diff --git a/Dockerfile b/bkapi/Dockerfile similarity index 64% rename from Dockerfile rename to bkapi/Dockerfile index 46e0e9d..5d1ce0f 100644 --- a/Dockerfile +++ b/bkapi/Dockerfile @@ -1,4 +1,4 @@ FROM debian:buster-slim EXPOSE 3000 -COPY ./bkapi /bin/bkapi +COPY ./bkapi/bkapi /bin/bkapi CMD ["/bin/bkapi"] diff --git a/src/main.rs b/bkapi/src/main.rs similarity index 100% rename from src/main.rs rename to bkapi/src/main.rs diff --git a/src/tree.rs b/bkapi/src/tree.rs similarity index 100% rename from src/tree.rs rename to bkapi/src/tree.rs