anthrove/bkapi
4
0
Fork 0
mirror of https://github.com/Syfaro/bkapi.git synced 2024-11-05 06:33:30 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add client.

Browse Source
This commit is contained in:
Syfaro 2021-07-28 15:43:13 -04:00
parent 3b3d1db2dc
commit 2ef64e6d70
9 changed files with 538 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitlab-ci.yml
View File

@ -11,7 +11,7 @@ build:
artifacts:
expire_in: 1 day
paths:
- ./bkapi
- ./bkapi/bkapi
cache:
- key:
files:
@ -20,8 +20,8 @@ build:
- target/
- .cargo/
script:
- cargo build --release --verbose
- mv ./target/release/bkapi ./bkapi
- cargo build --bin bkapi --release --verbose
- mv ./target/release/bkapi ./bkapi/bkapi
docker:
image:

340
Cargo.lock generated
View File

@ -337,6 +337,21 @@ dependencies = [
"tracing-unwrap",
]
[[package]]
name = "bkapi-client"
version = "0.1.0"
dependencies = [
"futures",
"opentelemetry",
"opentelemetry-http",
"reqwest",
"serde",
"serde_json",
"tokio",
"tracing",
"tracing-opentelemetry",
]
[[package]]
name = "block-buffer"
version = "0.9.0"
@ -449,6 +464,22 @@ dependencies = [
"version_check",
]
[[package]]
name = "core-foundation"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "core-foundation-sys"
version = "0.8.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"
[[package]]
name = "cpufeatures"
version = "0.1.5"
@ -623,6 +654,21 @@ version = "1.0.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
[[package]]
name = "foreign-types"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
dependencies = [
"foreign-types-shared",
]
[[package]]
name = "foreign-types-shared"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
version = "1.0.1"
@ -842,12 +888,66 @@ dependencies = [
"itoa",
]
[[package]]
name = "http-body"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
dependencies = [
"bytes",
"http",
"pin-project-lite",
]
[[package]]
name = "httparse"
version = "1.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"
[[package]]
name = "httpdate"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440"
[[package]]
name = "hyper"
version = "0.14.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11"
dependencies = [
"bytes",
"futures-channel",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
"httparse",
"httpdate",
"itoa",
"pin-project-lite",
"socket2",
"tokio",
"tower-service",
"tracing",
"want",
]
[[package]]
name = "hyper-tls"
version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [
"bytes",
"hyper",
"native-tls",
"tokio",
"tokio-native-tls",
]
[[package]]
name = "idna"
version = "0.2.3"
@ -884,6 +984,12 @@ version = "1.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f"
[[package]]
name = "ipnet"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9"
[[package]]
name = "itoa"
version = "0.4.7"
@ -1051,6 +1157,24 @@ dependencies = [
"winapi",
]
[[package]]
name = "native-tls"
version = "0.2.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4"
dependencies = [
"lazy_static",
"libc",
"log",
"openssl",
"openssl-probe",
"openssl-sys",
"schannel",
"security-framework",
"security-framework-sys",
"tempfile",
]
[[package]]
name = "nom"
version = "6.1.2"
@ -1114,6 +1238,39 @@ version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
[[package]]
name = "openssl"
version = "0.10.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885"
dependencies = [
"bitflags",
"cfg-if",
"foreign-types",
"libc",
"once_cell",
"openssl-sys",
]
[[package]]
name = "openssl-probe"
version = "0.1.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a"
[[package]]
name = "openssl-sys"
version = "0.9.65"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d"
dependencies = [
"autocfg",
"cc",
"libc",
"pkg-config",
"vcpkg",
]
[[package]]
name = "opentelemetry"
version = "0.15.0"
@ -1133,6 +1290,19 @@ dependencies = [
"tokio-stream",
]
[[package]]
name = "opentelemetry-http"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1eeb0d4b230c69aa425f9c3aadf02ce7aa4b2bc9a4e9fbe59614108232078263"
dependencies = [
"async-trait",
"bytes",
"futures-util",
"http",
"opentelemetry",
]
[[package]]
name = "opentelemetry-jaeger"
version = "0.14.0"
@ -1233,6 +1403,12 @@ version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
name = "pkg-config"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
[[package]]
name = "ppv-lite86"
version = "0.2.10"
@ -1397,6 +1573,50 @@ version = "0.6.25"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b"
[[package]]
name = "remove_dir_all"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7"
dependencies = [
"winapi",
]
[[package]]
name = "reqwest"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22"
dependencies = [
"base64",
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
"http",
"http-body",
"hyper",
"hyper-tls",
"ipnet",
"js-sys",
"lazy_static",
"log",
"mime",
"native-tls",
"percent-encoding",
"pin-project-lite",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-native-tls",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
"winreg",
]
[[package]]
name = "ring"
version = "0.16.20"
@ -1449,6 +1669,16 @@ version = "1.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
name = "schannel"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75"
dependencies = [
"lazy_static",
"winapi",
]
[[package]]
name = "scopeguard"
version = "1.1.0"
@ -1465,6 +1695,29 @@ dependencies = [
"untrusted",
]
[[package]]
name = "security-framework"
version = "2.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
dependencies = [
"bitflags",
"core-foundation",
"core-foundation-sys",
"libc",
"security-framework-sys",
]
[[package]]
name = "security-framework-sys"
version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
dependencies = [
"core-foundation-sys",
"libc",
]
[[package]]
name = "semver"
version = "0.9.0"
@ -1821,6 +2074,20 @@ version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369"
[[package]]
name = "tempfile"
version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
dependencies = [
"cfg-if",
"libc",
"rand",
"redox_syscall",
"remove_dir_all",
"winapi",
]
[[package]]
name = "thiserror"
version = "1.0.26"
@ -1941,9 +2208,31 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"tokio-macros",
"winapi",
]
[[package]]
name = "tokio-macros"
version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110"
dependencies = [
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "tokio-native-tls"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b"
dependencies = [
"native-tls",
"tokio",
]
[[package]]
name = "tokio-rustls"
version = "0.22.0"
@ -1980,6 +2269,12 @@ dependencies = [
"tokio",
]
[[package]]
name = "tower-service"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6"
[[package]]
name = "tracing"
version = "0.1.26"
@ -2108,6 +2403,12 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "622b09ce2fe2df4618636fb92176d205662f59803f39e70d1c333393082de96c"
[[package]]
name = "try-lock"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "typenum"
version = "1.13.0"
@ -2183,12 +2484,28 @@ dependencies = [
"getrandom",
]
[[package]]
name = "vcpkg"
version = "0.2.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
[[package]]
name = "version_check"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fecdca9a5291cc2b8dcf7dc02453fee791a280f3743cb0905f8822ae463b3fe"
[[package]]
name = "want"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ce8a968cb1cd110d136ff8b819a556d6fb6d919363c61534f6860c7eb172ba0"
dependencies = [
"log",
"try-lock",
]
[[package]]
name = "wasi"
version = "0.10.2+wasi-snapshot-preview1"
@ -2202,6 +2519,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
dependencies = [
"cfg-if",
"serde",
"serde_json",
"wasm-bindgen-macro",
]
@ -2220,6 +2539,18 @@ dependencies = [
"wasm-bindgen-shared",
]
[[package]]
name = "wasm-bindgen-futures"
version = "0.4.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
dependencies = [
"cfg-if",
"js-sys",
"wasm-bindgen",
"web-sys",
]
[[package]]
name = "wasm-bindgen-macro"
version = "0.2.74"
@ -2310,6 +2641,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
[[package]]
name = "winreg"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69"
dependencies = [
"winapi",
]
[[package]]
name = "wyz"
version = "0.2.0"

43
Cargo.toml
View File

@ -1,38 +1,5 @@
[package]
name = "bkapi"
version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"
[dependencies]
envconfig = "0.10"
thiserror = "1"
tracing = "0.1"
tracing-subscriber = "0.2"
tracing-unwrap = "0.9"
tracing-opentelemetry = "0.14"
opentelemetry = { version = "0.15", features = ["rt-tokio"] }
opentelemetry-jaeger = "0.14"
lazy_static = "1"
prometheus = { version = "0.12", features = ["process"] }
bk-tree = "0.4.0"
hamming = "0.1"
futures = "0.3"
tokio = { version = "1", features = ["sync"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
actix-web = "4.0.0-beta.8"
actix-http = "3.0.0-beta.8"
actix-service = "2"
tracing-actix-web = { version = "0.4.0-beta.9", features = ["opentelemetry_0_15"] }
[dependencies.sqlx]
version = "0.5"
features = ["runtime-actix-rustls", "postgres"]
[workspace]
members = [
"bkapi",
"bkapi-client",
]

22
bkapi-client/Cargo.toml Normal file
View File

@ -0,0 +1,22 @@
[package]
name = "bkapi-client"
version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"
[dependencies]
tracing = "0.1"
tracing-opentelemetry = "0.14"
opentelemetry = "0.15"
opentelemetry-http = "0.4"
futures = "0.3"
reqwest = { version = "0.11", features = ["json"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
[dev-dependencies]
tokio = { version = "1", features = ["full"] }

129
bkapi-client/src/lib.rs Normal file
View File

@ -0,0 +1,129 @@
use futures::TryStreamExt;
use serde::{Deserialize, Serialize};
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SearchResults {
pub hash: i64,
pub distance: u64,
pub hashes: Vec<SearchResult>,
}
#[derive(Clone, Debug, Serialize, Deserialize)]
pub struct SearchResult {
pub hash: i64,
pub distance: u64,
}
pub struct BKApiClient {
pub endpoint: String,
client: reqwest::Client,
}
impl BKApiClient {
pub fn new<E>(endpoint: E) -> Self
where
E: Into<String>,
{
Self {
endpoint: endpoint.into(),
client: reqwest::Client::default(),
}
}
#[tracing::instrument(err, skip(self))]
pub async fn search(&self, hash: i64, distance: u64) -> Result<SearchResults, reqwest::Error> {
let results = self
.client
.get(&self.endpoint)
.query(&[
("hash", hash.to_string()),
("distance", distance.to_string()),
])
.inject_context()
.send()
.await?
.error_for_status()?
.json()
.await?;
Ok(results)
}
#[tracing::instrument(err, skip(self))]
pub async fn search_many(
&self,
hashes: &[i64],
distance: u64,
) -> Result<Vec<SearchResults>, reqwest::Error> {
let mut futs = futures::stream::FuturesOrdered::new();
for hash in hashes {
futs.push(self.search(*hash, distance));
}
futs.try_collect().await
}
}
trait InjectContext {
fn inject_context(self) -> Self;
}
impl InjectContext for reqwest::RequestBuilder {
fn inject_context(self: reqwest::RequestBuilder) -> reqwest::RequestBuilder {
use tracing_opentelemetry::OpenTelemetrySpanExt;
let mut headers: reqwest::header::HeaderMap = Default::default();
let cx = tracing::Span::current().context();
opentelemetry::global::get_text_map_propagator(|propagator| {
propagator.inject_context(&cx, &mut opentelemetry_http::HeaderInjector(&mut headers))
});
self.headers(headers)
}
}
#[cfg(test)]
mod tests {
fn get_test_endpoint() -> String {
std::env::var("BKAPI_ENDPOINT").expect("tests require BKAPI_ENDPOINT to be set")
}
#[tokio::test]
async fn test_search() {
let endpoint = get_test_endpoint();
let client = super::BKApiClient::new(endpoint);
let results = client.search(0, 3).await;
println!("{:?}", results);
let results = results.expect("search should not have error");
assert_eq!(
0, results.hash,
"returned hash should have been {} but was {}",
0, results.hash
);
assert_eq!(
3, results.distance,
"returned distance should have been {} but was {}",
3, results.distance
);
}
#[tokio::test]
async fn test_search_many() {
let endpoint = get_test_endpoint();
let client = super::BKApiClient::new(endpoint);
let results = client.search_many(&[0, 1_000_000], 7).await;
println!("{:?}", results);
let results = results.expect("search should not have error");
assert_eq!(
2,
results.len(),
"returned results were missing search, expected {} items but found {} items",
2,
results.len()
);
}
}

38
bkapi/Cargo.toml Normal file
View File

@ -0,0 +1,38 @@
[package]
name = "bkapi"
version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"
[dependencies]
envconfig = "0.10"
thiserror = "1"
tracing = "0.1"
tracing-subscriber = "0.2"
tracing-unwrap = "0.9"
tracing-opentelemetry = "0.14"
opentelemetry = { version = "0.15", features = ["rt-tokio"] }
opentelemetry-jaeger = "0.14"
lazy_static = "1"
prometheus = { version = "0.12", features = ["process"] }
bk-tree = "0.4.0"
hamming = "0.1"
futures = "0.3"
tokio = { version = "1", features = ["sync"] }
serde = { version = "1", features = ["derive"] }
serde_json = "1"
actix-web = "4.0.0-beta.8"
actix-http = "3.0.0-beta.8"
actix-service = "2"
tracing-actix-web = { version = "0.4.0-beta.9", features = ["opentelemetry_0_15"] }
[dependencies.sqlx]
version = "0.5"
features = ["runtime-actix-rustls", "postgres"]

2
Dockerfile → bkapi/Dockerfile
View File

@ -1,4 +1,4 @@
FROM debian:buster-slim
EXPOSE 3000
COPY ./bkapi /bin/bkapi
COPY ./bkapi/bkapi /bin/bkapi
CMD ["/bin/bkapi"]

0
src/main.rs → bkapi/src/main.rs
View File

0
src/tree.rs → bkapi/src/tree.rs
View File