diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5c2d40f..9ef178a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,59 +4,32 @@ stages: - image variables: - CARGO_HOME: /root/.cargo - SCCACHE_VERSION: v0.2.15 - SCCACHE_S3_USE_SSL: "true" - SCCACHE_BUCKET: "sccache" - + CARGO_HOME: "$CI_PROJECT_DIR/.cargo" SQLX_OFFLINE: "true" # Cache should only be updated once, default to pull only cache: &global_cache + key: + files: + - Cargo.lock paths: - .cargo/ policy: pull # Run tests on current stable Rust version test:latest: &base_test - image: rust:1.51-slim-buster + image: rust:1.53-slim-buster stage: test cache: <<: *global_cache policy: pull-push before_script: - # Use proxy for apt, install wget to download sccache and other deps - - export http_proxy=$DEBIAN_PROXY - apt-get update -y - - apt-get install -y wget libssl-dev pkg-config libavcodec-dev libavformat-dev libavutil-dev libavdevice-dev clang llvm python3 python3-pip - - unset http_proxy - - # Download and extract sccache, ensuring it's executable - - wget -q -O /tmp/sccache.tar.gz https://github.com/mozilla/sccache/releases/download/$SCCACHE_VERSION/sccache-$SCCACHE_VERSION-x86_64-unknown-linux-musl.tar.gz - - tar zxvf /tmp/sccache.tar.gz --strip-components=1 -C /tmp - - export RUSTC_WRAPPER=/tmp/sccache - - chmod a+x $RUSTC_WRAPPER - - # Move sccache config into place - - mkdir -p $HOME/.config/sccache - - mv $SCCACHE_CONFIG $HOME/.config/sccache/config - - # Prepare cargo data from cache - - rm -rf /root/.cargo || true - - mv .cargo /root/.cargo || true - - # Prepare cargo-cache for cleaning data later - - cargo install cargo-cache --no-default-features --features ci-autoclean + - apt-get install -y libssl-dev pkg-config libavcodec-dev libavformat-dev libavutil-dev libavdevice-dev clang llvm python3 python3-pip script: # Build, test, and show stats - cargo build --verbose - cargo test --verbose - - $RUSTC_WRAPPER --show-stats - - # Clean cargo data, move back into place for caching - - $CARGO_HOME/bin/cargo-cache - - rm -rf .cargo || true - - mv /root/.cargo .cargo || true # Same as above, but nightly Rust test:nightly: @@ -77,7 +50,6 @@ build:api: &base_build - ./fuzzysearch/fuzzysearch script: - cargo build --verbose --release --bin fuzzysearch - - $RUSTC_WRAPPER --show-stats - mv ./target/release/fuzzysearch ./fuzzysearch/fuzzysearch build:webhook: @@ -88,9 +60,18 @@ build:webhook: - ./fuzzysearch-webhook/fuzzysearch-webhook script: - cargo build --verbose --release --bin fuzzysearch-webhook - - $RUSTC_WRAPPER --show-stats - mv ./target/release/fuzzysearch-webhook ./fuzzysearch-webhook/fuzzysearch-webhook +build:hash-input: + <<: *base_build + artifacts: + expire_in: 1 day + paths: + - ./fuzzysearch-hash-input/fuzzysearch-hash-input + script: + - cargo build --verbose --release --bin fuzzysearch-hash-input + - mv ./target/release/fuzzysearch-hash-input ./fuzzysearch-hash-input/fuzzysearch-hash-input + build:ingest-e621: <<: *base_build artifacts: @@ -99,7 +80,6 @@ build:ingest-e621: - ./fuzzysearch-ingest-e621/fuzzysearch-ingest-e621 script: - cargo build --verbose --release --bin fuzzysearch-ingest-e621 - - $RUSTC_WRAPPER --show-stats - mv ./target/release/fuzzysearch-ingest-e621 ./fuzzysearch-ingest-e621/fuzzysearch-ingest-e621 build:ingest-furaffinity: @@ -110,7 +90,6 @@ build:ingest-furaffinity: - ./fuzzysearch-ingest-furaffinity/fuzzysearch-ingest-furaffinity script: - cargo build --verbose --release --bin fuzzysearch-ingest-furaffinity - - $RUSTC_WRAPPER --show-stats - mv ./target/release/fuzzysearch-ingest-furaffinity ./fuzzysearch-ingest-furaffinity/fuzzysearch-ingest-furaffinity build:ingest-weasyl: @@ -121,54 +100,47 @@ build:ingest-weasyl: - ./fuzzysearch-ingest-weasyl/fuzzysearch-ingest-weasyl script: - cargo build --verbose --release --bin fuzzysearch-ingest-weasyl - - $RUSTC_WRAPPER --show-stats - mv ./target/release/fuzzysearch-ingest-weasyl ./fuzzysearch-ingest-weasyl/fuzzysearch-ingest-weasyl images:api: &base_images stage: image - image: docker + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] cache: {} before_script: - - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - mkdir -p /kaniko/.docker + - echo "{\"auths\":{\"$CI_REGISTRY\":{\"auth\":\"$(echo -n ${CI_REGISTRY_USER}:${CI_REGISTRY_PASSWORD} | base64)\"}}}" > /kaniko/.docker/config.json needs: ['build:api'] script: - - docker pull $CI_REGISTRY_IMAGE/api:latest || true - - docker build --build-arg http_proxy=$DEBIAN_PROXY --cache-from $CI_REGISTRY_IMAGE/api:latest --tag $CI_REGISTRY_IMAGE/api:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE/api:latest -f fuzzysearch/Dockerfile . - - docker push $CI_REGISTRY_IMAGE/api:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE/api:latest + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/fuzzysearch/Dockerfile --destination $CI_REGISTRY_IMAGE/api:$CI_COMMIT_SHA --destination $CI_REGISTRY_IMAGE/api:latest --cache=true images:webhook: <<: *base_images needs: ['build:webhook'] script: - - docker pull $CI_REGISTRY_IMAGE/webhook:latest || true - - docker build --build-arg http_proxy=$DEBIAN_PROXY --cache-from $CI_REGISTRY_IMAGE/webhook:latest --tag $CI_REGISTRY_IMAGE/webhook:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE/webhook:latest -f fuzzysearch-webhook/Dockerfile . - - docker push $CI_REGISTRY_IMAGE/webhook:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE/webhook:latest + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/fuzzysearch-webhook/Dockerfile --destination $CI_REGISTRY_IMAGE/webhook:$CI_COMMIT_SHA --destination $CI_REGISTRY_IMAGE/webhook:latest --cache=true + +images:hash-input: + <<: *base_images + needs: ['build:hash-input'] + script: + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/fuzzysearch-hash-input/Dockerfile --destination $CI_REGISTRY_IMAGE/hash-input:$CI_COMMIT_SHA --destination $CI_REGISTRY_IMAGE/hash-input:latest --cache=true images:ingest-e621: <<: *base_images needs: ['build:ingest-e621'] script: - - docker pull $CI_REGISTRY_IMAGE/ingest-e621:latest || true - - docker build --build-arg http_proxy=$DEBIAN_PROXY --cache-from $CI_REGISTRY_IMAGE/ingest-e621:latest --tag $CI_REGISTRY_IMAGE/ingest-e621:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE/ingest-e621:latest -f fuzzysearch-ingest-e621/Dockerfile . - - docker push $CI_REGISTRY_IMAGE/ingest-e621:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE/ingest-e621:latest + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/fuzzysearch-ingest-e621/Dockerfile --destination $CI_REGISTRY_IMAGE/ingest-e621:$CI_COMMIT_SHA --destination $CI_REGISTRY_IMAGE/ingest-e621:latest --cache=true images:ingest-furaffinity: <<: *base_images needs: ['build:ingest-furaffinity'] script: - - docker pull $CI_REGISTRY_IMAGE/ingest-furaffinity:latest || true - - docker build --build-arg http_proxy=$DEBIAN_PROXY --cache-from $CI_REGISTRY_IMAGE/ingest-furaffinity:latest --tag $CI_REGISTRY_IMAGE/ingest-furaffinity:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE/ingest-furaffinity:latest -f fuzzysearch-ingest-furaffinity/Dockerfile . - - docker push $CI_REGISTRY_IMAGE/ingest-furaffinity:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE/ingest-furaffinity:latest + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/fuzzysearch-ingest-furaffinity/Dockerfile --destination $CI_REGISTRY_IMAGE/ingest-furaffinity:$CI_COMMIT_SHA --destination $CI_REGISTRY_IMAGE/ingest-furaffinity:latest --cache=true images:ingest-weasyl: <<: *base_images needs: ['build:ingest-weasyl'] script: - - docker pull $CI_REGISTRY_IMAGE/ingest-weasyl:latest || true - - docker build --build-arg http_proxy=$DEBIAN_PROXY --cache-from $CI_REGISTRY_IMAGE/ingest-weasyl:latest --tag $CI_REGISTRY_IMAGE/ingest-weasyl:$CI_COMMIT_SHA --tag $CI_REGISTRY_IMAGE/ingest-weasyl:latest -f fuzzysearch-ingest-weasyl/Dockerfile . - - docker push $CI_REGISTRY_IMAGE/ingest-weasyl:$CI_COMMIT_SHA - - docker push $CI_REGISTRY_IMAGE/ingest-weasyl:latest + - /kaniko/executor --context $CI_PROJECT_DIR --dockerfile $CI_PROJECT_DIR/fuzzysearch-ingest-weasyl/Dockerfile --destination $CI_REGISTRY_IMAGE/ingest-weasyl:$CI_COMMIT_SHA --destination $CI_REGISTRY_IMAGE/ingest-weasyl:latest --cache=true diff --git a/Cargo.lock b/Cargo.lock index 42441b0..c5c82a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,10 +1,232 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + +[[package]] +name = "actix-codec" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d5dbeb2d9e51344cb83ca7cc170f1217f9fe25bfc50160e6e200b5c31c1019a" +dependencies = [ + "bitflags", + "bytes", + "futures-core", + "futures-sink", + "log", + "pin-project-lite", + "tokio", + "tokio-util", +] + +[[package]] +name = "actix-http" +version = "3.0.0-beta.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd16d6b846983ffabfd081e1a67abd7698094fcbe7b3d9bcf1acbc6f546a516" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-tls", + "actix-utils", + "ahash", + "base64", + "bitflags", + "brotli2", + "bytes", + "bytestring", + "derive_more", + "encoding_rs", + "flate2", + "futures-core", + "futures-util", + "h2", + "http", + "httparse", + "itoa", + "language-tags", + "local-channel", + "log", + "mime", + "once_cell", + "percent-encoding", + "pin-project", + "pin-project-lite", + "rand 0.8.4", + "regex", + "serde", + "sha-1", + "smallvec", + "time 0.2.27", + "tokio", + "zstd", +] + +[[package]] +name = "actix-macros" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2f86cd6857c135e6e9fe57b1619a88d1f94a7df34c00e11fe13e64fd3438837" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "actix-multipart" +version = "0.4.0-beta.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a32d8964e147f1e411b38cd08a28eb37915be6797191a394fe0ad73f36441a99" +dependencies = [ + "actix-utils", + "actix-web", + "bytes", + "derive_more", + "futures-core", + "futures-util", + "httparse", + "local-waker", + "log", + "mime", + "twoway 0.2.2", +] + +[[package]] +name = "actix-router" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ad299af73649e1fc893e333ccf86f377751eb95ff875d095131574c6f43452c" +dependencies = [ + "bytestring", + "http", + "log", + "regex", + "serde", +] + +[[package]] +name = "actix-rt" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d7cd957c9ed92288a7c3c96af81fa5291f65247a76a34dac7b6af74e52ba0" +dependencies = [ + "actix-macros", + "futures-core", + "tokio", +] + +[[package]] +name = "actix-server" +version = "2.0.0-beta.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26369215fcc3b0176018b3b68756a8bcc275bb000e6212e454944913a1f9bf87" +dependencies = [ + "actix-rt", + "actix-service", + "actix-utils", + "futures-core", + "log", + "mio", + "num_cpus", + "slab", + "tokio", +] + +[[package]] +name = "actix-service" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77f5f9d66a8730d0fae62c26f3424f5751e5518086628a40b7ab6fca4a705034" +dependencies = [ + "futures-core", + "paste 1.0.5", + "pin-project-lite", +] + +[[package]] +name = "actix-tls" +version = "3.0.0-beta.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65b7bb60840962ef0332f7ea01a57d73a24d2cb663708511ff800250bbfef569" +dependencies = [ + "actix-codec", + "actix-rt", + "actix-service", + "actix-utils", + "derive_more", + "futures-core", + "http", + "log", + "tokio-util", +] + +[[package]] +name = "actix-utils" +version = "3.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e491cbaac2e7fc788dfff99ff48ef317e23b3cf63dbaf7aaab6418f40f92aa94" +dependencies = [ + "local-waker", + "pin-project-lite", +] + +[[package]] +name = "actix-web" +version = "4.0.0-beta.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c503f726f895e55dac39adeafd14b5ee00cc956796314e9227fc7ae2e176f443" +dependencies = [ + "actix-codec", + "actix-http", + "actix-macros", + "actix-router", + "actix-rt", + "actix-server", + "actix-service", + "actix-utils", + "actix-web-codegen", + "ahash", + "bytes", + "cfg-if 1.0.0", + "cookie", + "derive_more", + "either", + "encoding_rs", + "futures-core", + "futures-util", + "itoa", + "language-tags", + "log", + "mime", + "once_cell", + "paste 1.0.5", + "pin-project", + "regex", + "serde", + "serde_json", + "serde_urlencoded", + "smallvec", + "socket2", + "time 0.2.27", + "url", +] + +[[package]] +name = "actix-web-codegen" +version = "0.5.0-beta.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d048c6986743105c1e8e9729fbc8d5d1667f2f62393a58be8d85a7d9a5a6c8d" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "addr2line" -version = "0.15.1" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03345e98af8f3d786b6d9f656ccfa6ac316d954e92bc4841f0bba20789d5fb5a" +checksum = "3e61f2b7f93d2c7d2b08263acaa4a363b3e276806c68af6134c44f523bf1aacd" dependencies = [ "gimli", ] @@ -23,17 +245,11 @@ checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" [[package]] name = "ahash" -version = "0.4.7" +version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" - -[[package]] -name = "ahash" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f200cbb1e856866d9eade941cf3aa0c5d7dd36f74311c4273b494f4ef036957" +checksum = "43bb833f0bf979d8475d38fbf09ed3b8a55e1885fe93ad3f93239fc6a4f17b98" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", "once_cell", "version_check", ] @@ -58,9 +274,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.40" +version = "1.0.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" +checksum = "595d3cfa7a60d4555cb5067b99f07142a08ea778de5cf993f7b75c7d8fabc486" [[package]] name = "arrayvec" @@ -68,27 +284,6 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" -[[package]] -name = "async-stream" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a26cb53174ddd320edfff199a853f93d571f48eeb4dde75e67a9a3dbb7b7e5e" -dependencies = [ - "async-stream-impl", - "futures-core", -] - -[[package]] -name = "async-stream-impl" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db134ba52475c060f3329a8ef0f8786d6b872ed01515d4b79c162e5798da1340" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "async-trait" version = "0.1.50" @@ -123,9 +318,9 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "backtrace" -version = "0.3.59" +version = "0.3.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4717cfcbfaa661a0fd48f8453951837ae7e8f81e481fbb136e3202d72805a744" +checksum = "e7a905d892734eea339e896738c14b9afce22b5318f64b951e70bf3844419b01" dependencies = [ "addr2line", "cc", @@ -136,6 +331,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base-x" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4521f3e3d031370679b3b140beb36dfe4801b09ac77e30c61941f97df3ef28b" + [[package]] name = "base64" version = "0.13.0" @@ -181,10 +382,19 @@ dependencies = [ ] [[package]] -name = "bk-tree" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5488039ea2c6de8668351415e39a0218a8955bffadcff0cf01d1293a20854584" +name = "bkapi-client" +version = "0.1.0" +source = "git+https://github.com/Syfaro/bkapi.git#04e494ee31139885e9e4d9cd71323e139e6baed7" +dependencies = [ + "futures", + "opentelemetry", + "opentelemetry-http", + "reqwest", + "serde", + "serde_json", + "tracing", + "tracing-opentelemetry", +] [[package]] name = "block-buffer" @@ -195,6 +405,26 @@ dependencies = [ "generic-array", ] +[[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" +dependencies = [ + "brotli-sys", + "libc", +] + [[package]] name = "buf_redux" version = "0.8.4" @@ -219,15 +449,15 @@ checksum = "b4ae4235e6dac0694637c763029ecea1a2ec9e4e06ec2729bd21ba4d9c863eb7" [[package]] name = "bumpalo" -version = "3.6.1" +version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63396b8a4b9de3f4fdfb320ab6080762242f66a8ef174c49d8e19b674db4cdbe" +checksum = "9c59e7af012c713f529e7a3ee57ce9b31ddd858d4b512923602f74608b009631" [[package]] name = "bytemuck" -version = "1.5.1" +version = "1.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bed57e2090563b83ba8f83366628ce535a7584c9afa4c9fc0612a03925c6df58" +checksum = "72957246c41db82b8ef88a5486143830adeb8227ef9837740bdec67724cf2c5b" [[package]] name = "byteorder" @@ -242,10 +472,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] -name = "cc" -version = "1.0.67" +name = "bytestring" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c69b077ad434294d3ce9f1f6143a2a4b89a8a2d54ef813d85003a4fd1137fd" +checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" +dependencies = [ + "bytes", +] + +[[package]] +name = "cc" +version = "1.0.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e70cc2f62c6ce1868963827bd677764c62d07c3d9a3e1fb1177ee1a9ab199eb2" +dependencies = [ + "jobserver", +] [[package]] name = "cexpr" @@ -286,7 +528,7 @@ dependencies = [ "num-integer", "num-traits", "serde", - "time", + "time 0.1.43", "winapi", ] @@ -307,12 +549,29 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "const_fn" +version = "0.4.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f92cfa0fd5690b3cf8c1ef2cabbd9b7ef22fa53cf5e1f92b05103f6d5d1cf6e7" + [[package]] name = "convert_case" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" +[[package]] +name = "cookie" +version = "0.15.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f1c7727e460397e56abc4bddc1d49e07a1ad78fc98eb2e1c8f032a58a2f80d" +dependencies = [ + "percent-encoding", + "time 0.2.27", + "version_check", +] + [[package]] name = "core-foundation" version = "0.9.1" @@ -331,9 +590,9 @@ checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" [[package]] name = "cpufeatures" -version = "0.1.1" +version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dec1028182c380cc45a2e2c5ec841134f2dfd0f8f5f0a5bcd68004f81b5efdf4" +checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef" dependencies = [ "libc", ] @@ -379,9 +638,9 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.4" +version = "0.9.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52fb27eab85b17fbb9f6fd667089e07d6a2eb8743d02639ee7f6a7a7729c9c94" +checksum = "4ec02e091aa634e2c3ada4a392989e7c3116673ef0ac5b72232439094d73b7fd" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -392,9 +651,9 @@ dependencies = [ [[package]] name = "crossbeam-queue" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" +checksum = "9b10ddc024425c88c2ad148c1b0fd53f4c6d38db9697c9f1588381212fa657c9" dependencies = [ "cfg-if 1.0.0", "crossbeam-utils", @@ -402,20 +661,19 @@ dependencies = [ [[package]] name = "crossbeam-utils" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4feb231f0d4d6af81aed15928e58ecf5816aa62a2393e2c82f46973e92a9a278" +checksum = "d82cfc11ce7f2c3faef78d8a684447b40d503d9681acebed6cb728d45940c4db" dependencies = [ - "autocfg", "cfg-if 1.0.0", "lazy_static", ] [[package]] name = "crypto-mac" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" +checksum = "bff07008ec701e8028e2ceb8f83f0e4274ee62bd2dbdc4fefff2e9a91824081a" dependencies = [ "generic-array", "subtle", @@ -470,13 +728,14 @@ dependencies = [ [[package]] name = "derive_more" -version = "0.99.13" +version = "0.99.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f82b1b72f1263f214c0f823371768776c4f5841b942c9883aa8e5ec584fd0ba6" +checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df" dependencies = [ "convert_case", "proc-macro2", "quote", + "rustc_version 0.3.3", "syn", ] @@ -509,6 +768,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "discard" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d0f5754cb6769937f4501cc0e67f4f4483c8d2c3e1e922ee9edbe4ab4c7c0" + [[package]] name = "dotenv" version = "0.15.0" @@ -589,7 +854,7 @@ dependencies = [ "fnv", "hostname", "libc", - "rand 0.8.3", + "rand 0.8.4", "serde", "serde_derive", "serde_json", @@ -605,9 +870,9 @@ checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" [[package]] name = "ffmpeg-next" -version = "4.3.8" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3186d58788057973c658e1fa13a26ea1340718a8a131d5e1996d06e2b08befc" +checksum = "4676cda947a87a1e8a42e154059c567e75de64860252cce52c684acd8c074fa0" dependencies = [ "bitflags", "ffmpeg-sys-next", @@ -616,9 +881,9 @@ dependencies = [ [[package]] name = "ffmpeg-sys-next" -version = "4.3.5" +version = "4.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fde8cbf91a1b044b86d9e9e944c33806a68f5e34e4281033594ceaab47a3746" +checksum = "de57234f2c49c6e093fe67bbbaa9142c228f6e2d5533ef27980993d5b6adef2a" dependencies = [ "bindgen", "cc", @@ -680,7 +945,7 @@ checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" [[package]] name = "furaffinity-rs" version = "0.1.0" -source = "git+https://github.com/Syfaro/furaffinity-rs#238d0b4516f3a11008284c31f991ba063926bb5b" +source = "git+https://github.com/Syfaro/furaffinity-rs?branch=main#9825e488e7c6737686e5f95dbcb56146aea6382e" dependencies = [ "cfscrape", "chrono", @@ -691,6 +956,7 @@ dependencies = [ "reqwest", "scraper", "sha2", + "thiserror", "tokio", ] @@ -706,9 +972,9 @@ dependencies = [ [[package]] name = "futures" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7e43a803dae2fa37c1f6a8fe121e1f7bf9548b4dfc0522a42f34145dadfc27" +checksum = "1adc00f486adfc9ce99f77d717836f0c5aa84965eb0b4f051f4e83f7cab53f8b" dependencies = [ "futures-channel", "futures-core", @@ -721,9 +987,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e682a68b29a882df0545c143dc3646daefe80ba479bcdede94d5a703de2871e2" +checksum = "74ed2411805f6e4e3d9bc904c95d5d423b89b3b25dc0250aa74729de20629ff9" dependencies = [ "futures-core", "futures-sink", @@ -731,15 +997,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0402f765d8a89a26043b889b26ce3c4679d268fa6bb22cd7c6aad98340e179d1" +checksum = "af51b1b4a7fdff033703db39de8802c673eb91855f2e0d47dcf3bf2c0ef01f99" [[package]] name = "futures-executor" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "badaa6a909fac9e7236d0620a2f57f7664640c56575b71a7552fbd68deafab79" +checksum = "4d0d535a57b87e1ae31437b892713aee90cd2d7b0ee48727cd11fc72ef54761c" dependencies = [ "futures-core", "futures-task", @@ -748,15 +1014,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acc499defb3b348f8d8f3f66415835a9131856ff7714bf10dadfc4ec4bdb29a1" +checksum = "0b0e06c393068f3a6ef246c75cdca793d6a46347e75286933e5e75fd2fd11582" [[package]] name = "futures-macro" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4c40298486cdf52cc00cd6d6987892ba502c7656a16a4192a9992b1ccedd121" +checksum = "c54913bae956fb8df7f4dc6fc90362aa72e69148e3f39041fbe8742d21e0ac57" dependencies = [ "autocfg", "proc-macro-hack", @@ -778,21 +1044,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a57bead0ceff0d6dde8f465ecd96c9338121bb7717d3e7b108059531870c4282" +checksum = "c0f30aaa67363d119812743aa5f33c201a7a66329f97d1a887022971feea4b53" [[package]] name = "futures-task" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a16bef9fc1a4dddb5bee51c989e3fbba26569cbb0e31f5b303c184e3dd33dae" +checksum = "bbe54a98670017f3be909561f6ad13e810d9a51f3f061b902062ca3da80799f2" [[package]] name = "futures-util" -version = "0.3.15" +version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb5c238d27e2bf94ffdfd27b2c29e3df4a68c4193bb6427384259e2bf191967" +checksum = "67eb846bfd58e44a8481a00049e82c43e0ccb5d61f8dc071057cb19249dd4d78" dependencies = [ "autocfg", "futures-channel", @@ -813,8 +1079,7 @@ dependencies = [ name = "fuzzysearch" version = "0.2.0" dependencies = [ - "async-stream", - "bk-tree", + "bkapi-client", "bytes", "chrono", "ffmpeg-next", @@ -824,9 +1089,8 @@ dependencies = [ "hyper", "image", "img_hash", - "infer", "lazy_static", - "opentelemetry 0.14.0", + "opentelemetry", "opentelemetry-http", "opentelemetry-jaeger", "prometheus", @@ -835,8 +1099,10 @@ dependencies = [ "serde_json", "sqlx", "tokio", + "tokio-stream", "tracing", "tracing-futures", + "tracing-log", "tracing-opentelemetry", "tracing-subscriber", "warp", @@ -850,23 +1116,50 @@ dependencies = [ "base64", "faktory", "ffmpeg-next", + "futures", + "hyper", "image", "img_hash", + "opentelemetry", + "opentelemetry-http", + "opentelemetry-jaeger", + "prometheus", + "reqwest", "serde", "serde_json", "tempfile", "tokio", "tracing", + "tracing-log", + "tracing-opentelemetry", "tracing-subscriber", ] +[[package]] +name = "fuzzysearch-hash-input" +version = "0.1.0" +dependencies = [ + "actix-http", + "actix-multipart", + "actix-web", + "anyhow", + "fuzzysearch-common", + "image", + "lazy_static", + "prometheus", + "tempfile", + "tokio", + "tokio-stream", + "tracing", + "tracing-actix-web", +] + [[package]] name = "fuzzysearch-ingest-e621" version = "0.1.0" dependencies = [ "anyhow", "fuzzysearch-common", - "hamming", "hyper", "image", "img_hash", @@ -894,10 +1187,7 @@ dependencies = [ "fuzzysearch-common", "hyper", "lazy_static", - "postgres", "prometheus", - "r2d2", - "r2d2_postgres", "reqwest", "serde", "serde_json", @@ -982,9 +1272,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ "cfg-if 1.0.0", "libc", @@ -1014,9 +1304,9 @@ dependencies = [ [[package]] name = "gimli" -version = "0.24.0" +version = "0.25.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4075386626662786ddb0ec9081e7c7eeb1ba31951f447ca780ef9f5d568189" +checksum = "f0a01e0497841a3b2db4f8afa483cce65f7e96a3498bd6c541734792aeac8fe7" [[package]] name = "glob" @@ -1051,18 +1341,18 @@ checksum = "65043da274378d68241eb9a8f8f8aa54e349136f7b8e12f63e3ef44043cc30e1" [[package]] name = "hashbrown" -version = "0.9.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" dependencies = [ - "ahash 0.4.7", + "ahash", ] [[package]] name = "hashlink" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +checksum = "7249a3129cbc1ffccd74857f81464a323a152173cdb134e0fd81bc803b29facf" dependencies = [ "hashbrown", ] @@ -1080,7 +1370,7 @@ dependencies = [ "http", "mime", "sha-1", - "time", + "time 0.1.43", ] [[package]] @@ -1094,18 +1384,18 @@ dependencies = [ [[package]] name = "heck" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +checksum = "6d621efb26863f0e9924c6ac577e8275e5e6b77455db64ffa6c65c904e9e132c" dependencies = [ "unicode-segmentation", ] [[package]] name = "hermit-abi" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" +checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33" dependencies = [ "libc", ] @@ -1181,15 +1471,15 @@ checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68" [[package]] name = "httpdate" -version = "1.0.0" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05842d0d43232b23ccb7060ecb0f0626922c21f30012e97b767b30afd4a5d4b9" +checksum = "6456b8a6c8f33fee7d958fcd1b60d55b11940a79e63ae87013e6d22e26034440" [[package]] name = "hyper" -version = "0.14.7" +version = "0.14.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e5f105c494081baa3bf9e200b279e27ec1623895cd504c7dbef8d0b080fcf54" +checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11" dependencies = [ "bytes", "futures-channel", @@ -1201,7 +1491,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project", + "pin-project-lite", "socket2", "tokio", "tower-service", @@ -1267,9 +1557,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.2" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824845a0bf897a9042383849b02c1bc219c2383772efcd5c6f9766fa4b81aef3" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ "autocfg", "hashbrown", @@ -1298,12 +1588,6 @@ dependencies = [ "unindent", ] -[[package]] -name = "infer" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f92b41dab759f9e8427c03f519c344a14655490b8db548dac1e57a75b3258391" - [[package]] name = "input_buffer" version = "0.4.0" @@ -1315,9 +1599,9 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +checksum = "bee0328b1209d157ef001c94dd85b4f8f64139adb0eac2659f4b08382b2f474d" dependencies = [ "cfg-if 1.0.0", ] @@ -1352,9 +1636,9 @@ dependencies = [ [[package]] name = "ipnet" -version = "2.3.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" +checksum = "68f2d64f2edebec4ce84ad108148e67e1064789bee435edc5b60ad398714a3a9" [[package]] name = "itoa" @@ -1362,6 +1646,15 @@ version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +[[package]] +name = "jobserver" +version = "0.1.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd" +dependencies = [ + "libc", +] + [[package]] name = "jpeg-decoder" version = "0.1.22" @@ -1380,6 +1673,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "language-tags" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4345964bb142484797b161f473a503a434de77149dd8c7427788c6e13379388" + [[package]] name = "lazy_static" version = "1.4.0" @@ -1407,9 +1706,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.94" +version = "0.2.98" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18794a8ad5b29321f790b55d93dfba91e125cb1a9edbd4f8e3150acc771c1a5e" +checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790" [[package]] name = "libloading" @@ -1421,6 +1720,24 @@ dependencies = [ "winapi", ] +[[package]] +name = "local-channel" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6246c68cf195087205a0512559c97e15eaf95198bf0e206d662092cdcb03fe9f" +dependencies = [ + "futures-core", + "futures-sink", + "futures-util", + "local-waker", +] + +[[package]] +name = "local-waker" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "84f9a2d3e27ce99ce2c3aad0b09b1a7b916293ea9b2bf624c13fe646fadd8da4" + [[package]] name = "lock_api" version = "0.4.4" @@ -1505,9 +1822,9 @@ checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memoffset" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d" +checksum = "59accc507f1338036a0477ef61afdae33cde60840f4dfe481319ce3ad116ddf9" dependencies = [ "autocfg", ] @@ -1549,9 +1866,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.11" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf80d3e903b34e0bd7282b218398aec54e082c840d9baf8339e0080a0c542956" +checksum = "8c2bdb6314ec10835cd3293dd268473a835c02b7b352e788be788b3c6ca6bb16" dependencies = [ "libc", "log", @@ -1584,7 +1901,7 @@ dependencies = [ "rand 0.7.3", "safemem", "tempfile", - "twoway", + "twoway 0.1.8", ] [[package]] @@ -1712,15 +2029,18 @@ dependencies = [ [[package]] name = "object" -version = "0.24.0" +version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a5b3dd1c072ee7963717671d1ca129f1048fda25edea6b752bfc71ac8854170" +checksum = "c55827317fb4c08822499848a14237d2874d6f139828893017237e7ab93eb386" +dependencies = [ + "memchr", +] [[package]] name = "once_cell" -version = "1.7.2" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af8b08b04175473088b46763e51ee54da5f9a164bc162f615b91bc179dbf15a3" +checksum = "692fcb63b64b1758029e0a96ee63e049ce8c5948587f2f7208df04625e5f6b56" [[package]] name = "opaque-debug" @@ -1730,9 +2050,9 @@ checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" [[package]] name = "openssl" -version = "0.10.34" +version = "0.10.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d7830286ad6a3973c0f1d9b73738f69c76b739301d0229c4b96501695cbe4c8" +checksum = "549430950c79ae24e6d02e0b7404534ecf311d94cc9f861e9e4020187d13d885" dependencies = [ "bitflags", "cfg-if 1.0.0", @@ -1750,9 +2070,9 @@ checksum = "28988d872ab76095a6e6ac88d99b54fd267702734fd7ffe610ca27f533ddb95a" [[package]] name = "openssl-sys" -version = "0.9.63" +version = "0.9.65" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6b0d6fb7d80f877617dfcb014e605e2b5ab2fb0afdf27935219bb6bd984cb98" +checksum = "7a7907e3bfa08bb85105209cdfcb6c63d109f8f6c1ed6ca318fff5c1853fbc1d" dependencies = [ "autocfg", "cc", @@ -1763,25 +2083,9 @@ dependencies = [ [[package]] name = "opentelemetry" -version = "0.13.0" +version = "0.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91cea1dfd50064e52db033179952d18c770cbc5dfefc8eba45d619357ba3914" -dependencies = [ - "async-trait", - "futures", - "js-sys", - "lazy_static", - "percent-encoding", - "pin-project", - "rand 0.8.3", - "thiserror", -] - -[[package]] -name = "opentelemetry" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "492848ff47f11b7f9de0443b404e2c5775f695e1af6b7076ca25f999581d547a" +checksum = "ff27b33e30432e7b9854936693ca103d8591b0501f7ae9f633de48cda3bf2a67" dependencies = [ "async-trait", "crossbeam-channel", @@ -1790,7 +2094,7 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project", - "rand 0.8.3", + "rand 0.8.4", "thiserror", "tokio", "tokio-stream", @@ -1798,26 +2102,26 @@ dependencies = [ [[package]] name = "opentelemetry-http" -version = "0.3.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b752610d59706ff7cae48003552b3d0bf19bcce532629c8a49095894f21c319" +checksum = "1eeb0d4b230c69aa425f9c3aadf02ce7aa4b2bc9a4e9fbe59614108232078263" dependencies = [ "async-trait", "bytes", "futures-util", "http", - "opentelemetry 0.14.0", + "opentelemetry", ] [[package]] name = "opentelemetry-jaeger" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fd9ed34f208e0394bfb17522ba0d890925685dfd883147670ed474339d4647" +checksum = "09a9fc8192722e7daa0c56e59e2336b797122fb8598383dcb11c8852733b435c" dependencies = [ "async-trait", "lazy_static", - "opentelemetry 0.14.0", + "opentelemetry", "thiserror", "thrift", "tokio", @@ -1867,6 +2171,12 @@ dependencies = [ "proc-macro-hack", ] +[[package]] +name = "paste" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf547ad0c65e31259204bd90935776d1c693cec2f4ff7abb7a1bbbd40dfe58" + [[package]] name = "paste-impl" version = "0.1.18" @@ -1888,6 +2198,15 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "pest" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53" +dependencies = [ + "ucd-trie", +] + [[package]] name = "phf" version = "0.8.0" @@ -1944,18 +2263,18 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7509cc106041c40a4518d2af7a61530e1eed0e6285296a3d8c5472806ccc4a4" +checksum = "576bc800220cc65dac09e99e97b08b358cfab6e17078de8dc5fee223bd2d0c08" dependencies = [ "pin-project-internal", ] [[package]] name = "pin-project-internal" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48c950132583b500556b1efd71d45b319029f2b71518d979fcc208e16b42426f" +checksum = "6e8fe8163d14ce7f0cdac2e040116f22eac817edabff0be91e8aff7e9accf389" dependencies = [ "proc-macro2", "quote", @@ -1964,9 +2283,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc0e1f259c92177c30a4c9d177246edd0a3568b25756a977d0632cf8fa37e905" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" [[package]] name = "pin-utils" @@ -2019,7 +2338,7 @@ dependencies = [ "hmac", "md-5", "memchr", - "rand 0.8.3", + "rand 0.8.4", "sha2", "stringprep", ] @@ -2062,9 +2381,9 @@ checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" -version = "1.0.26" +version = "1.0.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a152013215dca273577e18d2bf00fa862b89b24169fb78c4c95aeb07992c9cec" +checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612" dependencies = [ "unicode-xid", ] @@ -2102,9 +2421,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.23.0" +version = "2.24.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45604fc7a88158e7d514d8e22e14ac746081e7a70d7690074dd0029ee37458d6" +checksum = "db50e77ae196458ccd3dc58a31ea1a90b0698ab1b7928d89f644c25d72070267" [[package]] name = "pyo3" @@ -2117,7 +2436,7 @@ dependencies = [ "inventory", "libc", "parking_lot", - "paste", + "paste 0.1.18", "pyo3cls", "unindent", ] @@ -2202,14 +2521,14 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.3" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" +checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8" dependencies = [ "libc", - "rand_chacha 0.3.0", - "rand_core 0.6.2", - "rand_hc 0.3.0", + "rand_chacha 0.3.1", + "rand_core 0.6.3", + "rand_hc 0.3.1", ] [[package]] @@ -2224,12 +2543,12 @@ dependencies = [ [[package]] name = "rand_chacha" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" +checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" dependencies = [ "ppv-lite86", - "rand_core 0.6.2", + "rand_core 0.6.3", ] [[package]] @@ -2243,11 +2562,11 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" +checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", ] [[package]] @@ -2261,11 +2580,11 @@ dependencies = [ [[package]] name = "rand_hc" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" +checksum = "d51e9f596de227fda2ea6c84607f5558e196eeaf43c986b724ba4fb8fdf497e7" dependencies = [ - "rand_core 0.6.2", + "rand_core 0.6.3", ] [[package]] @@ -2279,9 +2598,9 @@ dependencies = [ [[package]] name = "rayon" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" dependencies = [ "autocfg", "crossbeam-deque", @@ -2291,9 +2610,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.9.0" +version = "1.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -2304,9 +2623,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" +checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee" dependencies = [ "bitflags", ] @@ -2317,7 +2636,7 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "528532f3d801c87aec9def2add9ca802fe569e44a544afe633765267840abe64" dependencies = [ - "getrandom 0.2.2", + "getrandom 0.2.3", "redox_syscall", ] @@ -2334,11 +2653,10 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.1.9" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae1ded71d66a4a97f5e961fd0cb25a5f366a42a41570d16a763a69c092c26ae4" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" dependencies = [ - "byteorder", "regex-syntax", ] @@ -2359,9 +2677,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.3" +version = "0.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2296f2fac53979e8ccbc4a1136b25dcefd37be9ed7e4a1f6b05a6029c84ff124" +checksum = "246e9f61b9bb77df069a947682be06e31ac43ea37862e244a69f177694ea6d22" dependencies = [ "base64", "bytes", @@ -2377,6 +2695,7 @@ dependencies = [ "lazy_static", "log", "mime", + "mime_guess", "native-tls", "percent-encoding", "pin-project-lite", @@ -2394,9 +2713,9 @@ dependencies = [ [[package]] name = "rustc-demangle" -version = "0.1.19" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "410f7acf3cb3a44527c5d9546bad4bf4e6c460915d5f9f2fc524498bfe8f70ce" +checksum = "dead70b0b5e03e9c814bcb6b01e03e68f7c57a80aa48c72ec92152ab3e818d49" [[package]] name = "rustc-hash" @@ -2404,6 +2723,24 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee" +dependencies = [ + "semver 0.11.0", +] + [[package]] name = "rustdct" version = "0.4.0" @@ -2493,9 +2830,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "2.2.0" +version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3670b1d2fdf6084d192bc71ead7aabe6c06aa2ea3fbd9cc3ac111fa5c2b1bd84" +checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467" dependencies = [ "bitflags", "core-foundation", @@ -2506,9 +2843,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3676258fd3cfe2c9a0ec99ce3038798d847ce3e4bb17746373eb9f0f1ac16339" +checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284" dependencies = [ "core-foundation-sys", "libc", @@ -2535,19 +2872,52 @@ dependencies = [ ] [[package]] -name = "serde" -version = "1.0.125" +name = "semver" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "558dc50e1a5a5fa7112ca2ce4effcb321b0300c0d4ccf0776a9f60cd89031171" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser 0.7.0", +] + +[[package]] +name = "semver" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6" +dependencies = [ + "semver-parser 0.10.2", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "semver-parser" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7" +dependencies = [ + "pest", +] + +[[package]] +name = "serde" +version = "1.0.126" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.125" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b093b7a2bb58203b5da3056c05b4ec1fed827dcfdb37347a8841695263b3d06d" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote", @@ -2590,9 +2960,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.5" +version = "0.9.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b659df5fc3ce22274daac600ffb845300bd2125bcfaec047823075afdab81c00" +checksum = "1a0c8611594e2ab4ebbf06ec7cbbf0a99450b8570e96cbf5188b5d5f6ef18d81" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2602,10 +2972,16 @@ dependencies = [ ] [[package]] -name = "sha2" -version = "0.9.4" +name = "sha1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f6b75b17576b792bef0db1bcc4b8b8bcdf9506744cf34b974195487af6cff2" +checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" + +[[package]] +name = "sha2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12" dependencies = [ "block-buffer", "cfg-if 1.0.0", @@ -2631,18 +3007,18 @@ checksum = "7fdf1b9db47230893d76faad238fd6097fd6d6a9245cd7a4d90dbd639536bbd2" [[package]] name = "signal-hook-registry" -version = "1.3.0" +version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +checksum = "e51e73328dc4ac0c7ccbda3a494dfa03df1de2f46018127f60c693f2648455b0" dependencies = [ "libc", ] [[package]] name = "siphasher" -version = "0.3.5" +version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbce6d4507c7e4a3962091436e56e95290cb71fa302d0d270e32130b75fbff27" +checksum = "729a25c17d72b06c68cb47955d44fda88ad2d3e7d77e025663fdd69b93dd71a1" [[package]] name = "slab" @@ -2658,9 +3034,9 @@ checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "socket2" -version = "0.4.0" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +checksum = "765f090f0e423d2b55843402a07915add955e7d60657db13707a159727326cad" dependencies = [ "libc", "winapi", @@ -2681,9 +3057,9 @@ dependencies = [ [[package]] name = "sqlx" -version = "0.5.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d582b9bc04ec6c03084196efc42c2226b018e9941f03ee62bd88921d500917c0" +checksum = "ba82f79b31f30acebf19905bcd8b978f46891b9d0723f578447361a8910b6584" dependencies = [ "sqlx-core", "sqlx-macros", @@ -2691,11 +3067,11 @@ dependencies = [ [[package]] name = "sqlx-core" -version = "0.5.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de52d1d473cebb2abb79c886ef6a8023e965e34c0676a99cfeac2cc7f0fde4c1" +checksum = "7f23af36748ec8ea8d49ef8499839907be41b0b1178a4e82b8cb45d29f531dc9" dependencies = [ - "ahash 0.7.2", + "ahash", "atoi", "base64", "bitflags", @@ -2721,7 +3097,7 @@ dependencies = [ "once_cell", "parking_lot", "percent-encoding", - "rand 0.8.3", + "rand 0.8.4", "serde", "serde_json", "sha-1", @@ -2738,9 +3114,9 @@ dependencies = [ [[package]] name = "sqlx-macros" -version = "0.5.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a40f0be97e704d3fbf059e7e3333c3735639146a72d586c5534c70e79da88a4" +checksum = "47e4a2349d1ffd60a03ca0de3f116ba55d7f406e55a0d84c64a5590866d94c06" dependencies = [ "dotenv", "either", @@ -2761,9 +3137,9 @@ dependencies = [ [[package]] name = "sqlx-rt" -version = "0.5.2" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6ae97ab05063ed515cdc23d90253213aa24dda0a288c5ec079af3d10f9771bc" +checksum = "8199b421ecf3493ee9ef3e7bc90c904844cfb2ea7ea2f57347a93f52bfd3e057" dependencies = [ "native-tls", "once_cell", @@ -2777,12 +3153,70 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "standback" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e113fb6f3de07a243d434a56ec6f186dfd51cb08448239fe7bcae73f87ff28ff" +dependencies = [ + "version_check", +] + [[package]] name = "static_assertions" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +[[package]] +name = "stdweb" +version = "0.4.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5" +dependencies = [ + "discard", + "rustc_version 0.2.3", + "stdweb-derive", + "stdweb-internal-macros", + "stdweb-internal-runtime", + "wasm-bindgen", +] + +[[package]] +name = "stdweb-derive" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c87a60a40fccc84bef0652345bbbbbe20a605bf5d0ce81719fc476f5c03b50ef" +dependencies = [ + "proc-macro2", + "quote", + "serde", + "serde_derive", + "syn", +] + +[[package]] +name = "stdweb-internal-macros" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fa5ff6ad0d98d1ffa8cb115892b6e69d67799f6763e162a1c9db421dc22e11" +dependencies = [ + "base-x", + "proc-macro2", + "quote", + "serde", + "serde_derive", + "serde_json", + "sha1", + "syn", +] + +[[package]] +name = "stdweb-internal-runtime" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213701ba3370744dcd1a12960caa4843b3d68b4d1c0a5d575e0d65b2ee9d16c0" + [[package]] name = "strength_reduce" version = "0.2.3" @@ -2826,15 +3260,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" +checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" [[package]] name = "syn" -version = "1.0.72" +version = "1.0.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" +checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" dependencies = [ "proc-macro2", "quote", @@ -2843,9 +3277,9 @@ dependencies = [ [[package]] name = "synstructure" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b834f2d66f734cb897113e34aaff2f1ab4719ca946f9a7358dba8f8064148701" +checksum = "474aaa926faa1603c40b7885a9eaea29b444d1cb2850cb7c0e37bb1a4182f4fa" dependencies = [ "proc-macro2", "quote", @@ -2867,7 +3301,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.3", + "rand 0.8.4", "redox_syscall", "remove_dir_all", "winapi", @@ -2892,18 +3326,18 @@ checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" [[package]] name = "thiserror" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0f4a65597094d4483ddaed134f409b2cb7c1beccf25201a9f73c719254fa98e" +checksum = "93119e4feac1cbe6c798c34d3a53ea0026b0b1de6a120deef895137c0529bfe2" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.24" +version = "1.0.26" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7765189610d8241a44529806d6fd1f2e0a08734313a35d5b3a556f92b381f3c0" +checksum = "060d69a0afe7796bf42e9e2ff91f5ee691fb15c53d38b4b62a9a53eb23164745" dependencies = [ "proc-macro2", "quote", @@ -2963,10 +3397,48 @@ dependencies = [ ] [[package]] -name = "tinyvec" -version = "1.2.0" +name = "time" +version = "0.2.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" +checksum = "4752a97f8eebd6854ff91f1c1824cd6160626ac4bd44287f7f4ea2035a02a242" +dependencies = [ + "const_fn", + "libc", + "standback", + "stdweb", + "time-macros", + "version_check", + "winapi", +] + +[[package]] +name = "time-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "957e9c6e26f12cb6d0dd7fc776bb67a706312e7299aed74c8dd5b17ebb27e2f1" +dependencies = [ + "proc-macro-hack", + "time-macros-impl", +] + +[[package]] +name = "time-macros-impl" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3c141a1b43194f3f56a1411225df8646c55781d5f26db825b3d98507eb482f" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "standback", + "syn", +] + +[[package]] +name = "tinyvec" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "848a1e1181b9f6753b5e96a092749e29b11d19ede67dfbbd6c7dc7e0f49b5338" dependencies = [ "tinyvec_macros", ] @@ -2979,9 +3451,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.5.0" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83f0c8e7c0addab50b663055baf787d0af7f413a46e6e7fb9559a4e4db7137a5" +checksum = "4b7b349f11a7047e6d1276853e612d152f5e8a352c61917887cc2169e2366b4c" dependencies = [ "autocfg", "bytes", @@ -2999,9 +3471,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.1.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" +checksum = "54473be61f4ebe4efd09cec9bd5d16fa51d70ea0192213d754d2d500457db110" dependencies = [ "proc-macro2", "quote", @@ -3043,9 +3515,9 @@ dependencies = [ [[package]] name = "tokio-stream" -version = "0.1.5" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e177a5d8c3bf36de9ebe6d58537d8879e964332f93fb3339e43f618c81361af0" +checksum = "7b2f3f698253f03119ac0102beaa64f67a67e08074d03a22d18784104543727f" dependencies = [ "futures-core", "pin-project-lite", @@ -3067,9 +3539,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.6.6" +version = "0.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "940a12c99365c31ea8dd9ba04ec1be183ffe4920102bb7122c2f515437601e8e" +checksum = "1caa0b0c8d94a049db56b5acf8cba99dc0623aab1b26d5b5f5e2d945846b3592" dependencies = [ "bytes", "futures-core", @@ -3098,6 +3570,21 @@ dependencies = [ "tracing-core", ] +[[package]] +name = "tracing-actix-web" +version = "0.4.0-beta.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c794ee03ca18c0d149e6928db480700644ab405f553f55dc0650f541e73dc180" +dependencies = [ + "actix-web", + "futures", + "opentelemetry", + "tracing", + "tracing-futures", + "tracing-opentelemetry", + "uuid", +] + [[package]] name = "tracing-attributes" version = "0.1.15" @@ -3141,11 +3628,11 @@ dependencies = [ [[package]] name = "tracing-opentelemetry" -version = "0.12.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99003208b647dae59dcefc49c98aecaa3512fbc29351685d4b9ef23a9218458e" +checksum = "c47440f2979c4cd3138922840eec122e3c0ba2148bc290f756bd7fd60fc97fff" dependencies = [ - "opentelemetry 0.13.0", + "opentelemetry", "tracing", "tracing-core", "tracing-log", @@ -3164,9 +3651,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.2.18" +version = "0.2.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa5553bf0883ba7c9cbe493b085c29926bd41b66afc31ff72cf17ff4fb60dcd5" +checksum = "ab69019741fca4d98be3c62d2b75254528b5432233fd8a4d2739fec20278de48" dependencies = [ "ansi_term", "chrono", @@ -3228,7 +3715,7 @@ dependencies = [ "httparse", "input_buffer", "log", - "rand 0.8.3", + "rand 0.8.4", "sha-1", "url", "utf-8", @@ -3243,12 +3730,34 @@ dependencies = [ "memchr", ] +[[package]] +name = "twoway" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c57ffb460d7c24cd6eda43694110189030a3d1dfe418416d9468fd1c1d290b47" +dependencies = [ + "memchr", + "unchecked-index", +] + [[package]] name = "typenum" version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879f6906492a7cd215bfa4cf595b600146ccfac0c79bcbd1f3000162af5e8b06" +[[package]] +name = "ucd-trie" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c" + +[[package]] +name = "unchecked-index" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c" + [[package]] name = "unicase" version = "2.6.0" @@ -3269,18 +3778,18 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.17" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" +checksum = "d54590932941a9e9266f0832deed84ebe1bf2e4c9e4a3554d393d18f5e854bf9" dependencies = [ "tinyvec", ] [[package]] name = "unicode-segmentation" -version = "1.7.1" +version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" +checksum = "8895849a949e7845e06bd6dc1aa51731a103c42707010a5b591c0038fb73385b" [[package]] name = "unicode-width" @@ -3325,10 +3834,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" [[package]] -name = "vcpkg" -version = "0.2.12" +name = "uuid" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbdbff6266a24120518560b5dc983096efb98462e51d0d68169895b237be3e5d" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" +dependencies = [ + "getrandom 0.2.3", +] + +[[package]] +name = "vcpkg" +version = "0.2.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" [[package]] name = "version_check" @@ -3517,3 +4035,32 @@ name = "wyz" version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" + +[[package]] +name = "zstd" +version = "0.7.0+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9428752481d8372e15b1bf779ea518a179ad6c771cca2d2c60e4fbff3cc2cd52" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "3.1.0+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aa1926623ad7fe406e090555387daf73db555b948134b4d73eac5eb08fb666d" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "1.5.0+zstd.1.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e6c094340240369025fc6b731b054ee2a834328fa584310ac96aa4baebdc465" +dependencies = [ + "cc", + "libc", +] diff --git a/Cargo.toml b/Cargo.toml index fd6bdb5..1520568 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,18 +1,16 @@ [workspace] members = [ - "fuzzysearch", "fuzzysearch-common", + + "fuzzysearch", + "fuzzysearch-hash-input", + "fuzzysearch-webhook", + "fuzzysearch-ingest-e621", "fuzzysearch-ingest-furaffinity", "fuzzysearch-ingest-weasyl", - "fuzzysearch-webhook" ] -[profile.dev] +[profile.dev.package."*"] opt-level = 2 debug = true - -[profile.release] -lto = true -codegen-units = 1 -opt-level = 3 diff --git a/fuzzysearch-common/Cargo.toml b/fuzzysearch-common/Cargo.toml index 32021b8..512a225 100644 --- a/fuzzysearch-common/Cargo.toml +++ b/fuzzysearch-common/Cargo.toml @@ -5,19 +5,25 @@ authors = ["Syfaro "] edition = "2018" [features] -default = [] +default = ["trace"] video = ["ffmpeg-next", "tempfile"] queue = ["faktory", "tokio", "serde_json"] +trace = ["opentelemetry", "opentelemetry-jaeger", "tracing-opentelemetry", "opentelemetry-http", "hyper", "prometheus", "tokio", "reqwest"] [dependencies] anyhow = "1" tracing = "0.1" tracing-subscriber = "0.2" +tracing-log = "0.1" + +tokio = { version = "1", features = ["rt"], optional = true } +futures = "0.3" serde = { version = "1", features = ["derive"] } -base64 = "0.13" +serde_json = { version = "1", optional = true } +base64 = "0.13" image = "0.23" img_hash = "3" @@ -25,5 +31,12 @@ ffmpeg-next = { version = "4", optional = true } tempfile = { version = "3", optional = true } faktory = { version = "0.11", optional = true } -tokio = { version = "1", features = ["rt"], optional = true } -serde_json = { version = "1", optional = true } + +opentelemetry = { version = "0.15.0", features = ["rt-tokio"], optional = true } +opentelemetry-jaeger = { version = "0.14", features = ["tokio"], optional = true } +tracing-opentelemetry = { version = "0.14", optional = true } +opentelemetry-http = { version = "0.4", optional = true } + +hyper = { version = "0.14", features = ["server", "http2", "tcp"], optional = true } +prometheus = { version = "0.12", optional = true } +reqwest = { version = "0.11", optional = true } diff --git a/fuzzysearch-common/src/lib.rs b/fuzzysearch-common/src/lib.rs index e0ec059..0a7d606 100644 --- a/fuzzysearch-common/src/lib.rs +++ b/fuzzysearch-common/src/lib.rs @@ -4,6 +4,9 @@ pub mod types; #[cfg(feature = "video")] pub mod video; +#[cfg(feature = "trace")] +pub mod trace; + /// Create an instance of img_hash with project defaults. pub fn get_hasher() -> img_hash::Hasher<[u8; 8]> { use img_hash::{HashAlg::Gradient, HasherConfig}; diff --git a/fuzzysearch-common/src/trace.rs b/fuzzysearch-common/src/trace.rs new file mode 100644 index 0000000..043cf1e --- /dev/null +++ b/fuzzysearch-common/src/trace.rs @@ -0,0 +1,118 @@ +pub fn configure_tracing(service_name: &'static str) { + use opentelemetry::KeyValue; + use tracing_subscriber::layer::SubscriberExt; + + tracing_log::LogTracer::init().unwrap(); + + let env = std::env::var("ENVIRONMENT"); + let env = if let Ok(env) = env.as_ref() { + env.as_str() + } else if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + + opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); + + let tracer = opentelemetry_jaeger::new_pipeline() + .with_agent_endpoint(std::env::var("JAEGER_COLLECTOR").expect("Missing JAEGER_COLLECTOR")) + .with_service_name(service_name) + .with_tags(vec![ + KeyValue::new("environment", env.to_owned()), + KeyValue::new("version", env!("CARGO_PKG_VERSION")), + ]) + .install_batch(opentelemetry::runtime::Tokio) + .unwrap(); + + let trace = tracing_opentelemetry::layer().with_tracer(tracer); + let env_filter = tracing_subscriber::EnvFilter::from_default_env(); + + if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) { + let subscriber = tracing_subscriber::fmt::layer() + .json() + .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339()) + .with_target(true); + let subscriber = tracing_subscriber::Registry::default() + .with(env_filter) + .with(trace) + .with(subscriber); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } else { + let subscriber = tracing_subscriber::fmt::layer(); + let subscriber = tracing_subscriber::Registry::default() + .with(env_filter) + .with(trace) + .with(subscriber); + tracing::subscriber::set_global_default(subscriber).unwrap(); + } + + tracing::debug!(service_name, "set application tracing service name"); +} + +async fn metrics( + req: hyper::Request, +) -> Result, std::convert::Infallible> { + use hyper::{Body, Response, StatusCode}; + + match req.uri().path() { + "/health" => Ok(Response::new(Body::from("OK"))), + "/metrics" => { + use prometheus::{Encoder, TextEncoder}; + + let mut buffer = Vec::new(); + let encoder = TextEncoder::new(); + + let metric_families = prometheus::gather(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + Ok(Response::new(Body::from(buffer))) + } + _ => { + let mut not_found = Response::new(Body::default()); + *not_found.status_mut() = StatusCode::NOT_FOUND; + Ok(not_found) + } + } +} + +pub async fn serve_metrics() { + use hyper::{ + server::Server, + service::{make_service_fn, service_fn}, + }; + use std::convert::Infallible; + use std::net::SocketAddr; + + let make_svc = make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(metrics)) }); + + let addr: SocketAddr = std::env::var("METRICS_HOST") + .expect("Missing METRICS_HOST") + .parse() + .expect("Invalid METRICS_HOST"); + + let server = Server::bind(&addr).serve(make_svc); + + tokio::spawn(async move { + server.await.expect("Metrics server error"); + }); +} + +pub trait InjectContext { + fn inject_context(self) -> Self; +} + +impl InjectContext for reqwest::RequestBuilder { + fn inject_context(self: reqwest::RequestBuilder) -> reqwest::RequestBuilder { + use tracing_opentelemetry::OpenTelemetrySpanExt; + + let mut headers: reqwest::header::HeaderMap = Default::default(); + + let cx = tracing::Span::current().context(); + opentelemetry::global::get_text_map_propagator(|propagator| { + propagator.inject_context(&cx, &mut opentelemetry_http::HeaderInjector(&mut headers)) + }); + + self.headers(headers) + } +} diff --git a/fuzzysearch-hash-input/Cargo.toml b/fuzzysearch-hash-input/Cargo.toml new file mode 100644 index 0000000..7a478b3 --- /dev/null +++ b/fuzzysearch-hash-input/Cargo.toml @@ -0,0 +1,25 @@ +[package] +name = "fuzzysearch-hash-input" +version = "0.1.0" +authors = ["Syfaro "] +edition = "2018" + +[dependencies] +tracing = "0.1" +anyhow = "1" + +tokio = { version = "1", features = ["full"] } +tokio-stream = "0.1" + +tempfile = "3" +image = "0.23" + +actix-web = "4.0.0-beta.5" +actix-http = "3.0.0-beta.5" +actix-multipart = "0.4.0-beta.4" +tracing-actix-web = { version = "0.4.0-beta.4", features = ["opentelemetry_0_15"] } + +lazy_static = "1" +prometheus = { version = "0.12", features = ["process"] } + +fuzzysearch-common = { path = "../fuzzysearch-common" } diff --git a/fuzzysearch-hash-input/Dockerfile b/fuzzysearch-hash-input/Dockerfile new file mode 100644 index 0000000..f63e80f --- /dev/null +++ b/fuzzysearch-hash-input/Dockerfile @@ -0,0 +1,4 @@ +FROM debian:buster-slim +RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* +COPY ./fuzzysearch-hash-input/fuzzysearch-hash-input /bin/fuzzysearch-hash-input +CMD ["/bin/fuzzysearch-hash-input"] diff --git a/fuzzysearch-hash-input/src/main.rs b/fuzzysearch-hash-input/src/main.rs new file mode 100644 index 0000000..3dcbd55 --- /dev/null +++ b/fuzzysearch-hash-input/src/main.rs @@ -0,0 +1,140 @@ +use std::{ + convert::TryInto, + io::{BufReader, SeekFrom}, +}; + +use actix_web::{post, web::Data, App, HttpRequest, HttpResponse, HttpServer, Responder}; +use tempfile::tempfile; +use tokio::{ + io::{AsyncSeekExt, AsyncWriteExt}, + sync::Semaphore, +}; +use tokio_stream::StreamExt; + +lazy_static::lazy_static! { + static ref IMAGE_LOADING_DURATION: prometheus::Histogram = + prometheus::register_histogram!("fuzzysearch_image_image_loading_seconds", "Duration to download and save image").unwrap(); + static ref IMAGE_DECODING_DURATION: prometheus::Histogram = + prometheus::register_histogram!("fuzzysearch_image_image_decoding_seconds", "Duration to decode image data").unwrap(); + static ref IMAGE_HASHING_DURATION: prometheus::Histogram = + prometheus::register_histogram!("fuzzysearch_image_image_hashing_seconds", "Duration to hash image").unwrap(); +} + +enum ImageResponse { + Hash(i64), + Error(anyhow::Error), +} + +impl Responder for ImageResponse { + fn respond_to(self, _req: &HttpRequest) -> HttpResponse { + match self { + ImageResponse::Hash(hash) => HttpResponse::Ok() + .content_type("text/plain") + .body(hash.to_string()), + ImageResponse::Error(error) => HttpResponse::BadRequest() + .content_type("text/plain") + .body(error.to_string()), + } + } +} + +#[tracing::instrument(err, skip(field, semaphore))] +async fn process_image( + mut field: actix_multipart::Field, + semaphore: Data, +) -> anyhow::Result { + tracing::debug!("creating temp file"); + + let loading_duration = IMAGE_LOADING_DURATION.start_timer(); + let mut file = + tokio::task::spawn_blocking(move || -> anyhow::Result { + let file = tempfile()?; + Ok(tokio::fs::File::from_std(file)) + }) + .await??; + + tracing::debug!("writing contents to temp file"); + let mut size = 0; + while let Ok(Some(chunk)) = field.try_next().await { + file.write_all(&chunk).await?; + size += chunk.len(); + } + tracing::debug!("file was {} bytes", size); + + tracing::debug!("returning file to beginning"); + file.seek(SeekFrom::Start(0)).await?; + let file = file.into_std().await; + loading_duration.stop_and_record(); + + tracing::debug!("getting semaphore permit"); + let _permit = semaphore.acquire().await?; + + tracing::debug!("decoding and hashing image"); + let hash = tokio::task::spawn_blocking(move || -> anyhow::Result { + let decoding_duration = IMAGE_DECODING_DURATION.start_timer(); + let reader = BufReader::new(file); + let reader = image::io::Reader::new(reader).with_guessed_format()?; + let im = reader.decode()?; + decoding_duration.stop_and_record(); + + let hashing_duration = IMAGE_HASHING_DURATION.start_timer(); + let image_hash = fuzzysearch_common::get_hasher().hash_image(&im); + let hash: [u8; 8] = image_hash.as_bytes().try_into()?; + let hash = i64::from_be_bytes(hash); + hashing_duration.stop_and_record(); + + Ok(hash) + }) + .await??; + + tracing::debug!("calculated image hash: {}", hash); + Ok(hash) +} + +#[post("/image")] +async fn post_image( + mut form: actix_multipart::Multipart, + semaphore: Data, +) -> impl Responder { + while let Ok(Some(field)) = form.try_next().await { + tracing::debug!("got multipart field: {:?}", field); + + let content_type = if let Some(content_disposition) = field.content_disposition() { + content_disposition + } else { + continue; + }; + + if !matches!(content_type.get_name(), Some("image")) { + continue; + } + + match process_image(field, semaphore).await { + Ok(hash) => return ImageResponse::Hash(hash), + Err(err) => return ImageResponse::Error(err), + } + } + + ImageResponse::Error(anyhow::anyhow!("missing image field")) +} + +#[actix_web::main] +async fn main() { + fuzzysearch_common::trace::configure_tracing("fuzzysearch-image"); + fuzzysearch_common::trace::serve_metrics().await; + + let semaphore = Data::new(Semaphore::new(4)); + + HttpServer::new(move || { + App::new() + .wrap(tracing_actix_web::TracingLogger::default()) + .app_data(semaphore.clone()) + .service(post_image) + }) + .workers(2) + .bind("0.0.0.0:8090") + .unwrap() + .run() + .await + .unwrap(); +} diff --git a/fuzzysearch-ingest-e621/Cargo.toml b/fuzzysearch-ingest-e621/Cargo.toml index fc4aed4..86bb43a 100644 --- a/fuzzysearch-ingest-e621/Cargo.toml +++ b/fuzzysearch-ingest-e621/Cargo.toml @@ -16,7 +16,6 @@ serde_json = "1" sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "macros", "json", "offline"] } image = "0.23" -hamming = "0.1.3" img_hash = "3" sha2 = "0.9" diff --git a/fuzzysearch-ingest-e621/Dockerfile b/fuzzysearch-ingest-e621/Dockerfile index 9fb4346..8e9bec2 100644 --- a/fuzzysearch-ingest-e621/Dockerfile +++ b/fuzzysearch-ingest-e621/Dockerfile @@ -1,14 +1,6 @@ -FROM rust:1-slim-buster AS builder -WORKDIR /src -ENV SQLX_OFFLINE=true -RUN apt-get update -y && apt-get install -y libssl-dev pkg-config -COPY . . -RUN cargo install --root / --path ./fuzzysearch-ingest-e621 - FROM debian:buster-slim EXPOSE 8080 ENV METRICS_HOST=0.0.0.0:8080 -WORKDIR /app RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* -COPY --from=builder /bin/fuzzysearch-ingest-e621 /bin/fuzzysearch-ingest-e621 +COPY ./fuzzysearch-ingest-e621/fuzzysearch-ingest-e621 /bin/fuzzysearch-ingest-e621 CMD ["/bin/fuzzysearch-ingest-e621"] diff --git a/fuzzysearch-ingest-e621/src/main.rs b/fuzzysearch-ingest-e621/src/main.rs index e807480..39fcf2b 100644 --- a/fuzzysearch-ingest-e621/src/main.rs +++ b/fuzzysearch-ingest-e621/src/main.rs @@ -30,9 +30,8 @@ type Auth = (String, Option); #[tokio::main] async fn main() -> anyhow::Result<()> { - fuzzysearch_common::init_logger(); - - create_metrics_server().await; + fuzzysearch_common::trace::configure_tracing("fuzzysearch-ingest-e621"); + fuzzysearch_common::trace::serve_metrics().await; let login = std::env::var("E621_LOGIN").expect_or_log("Missing E621_LOGIN"); let api_key = std::env::var("E621_API_KEY").expect_or_log("Missing E621_API_KEY"); @@ -345,41 +344,3 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result, -) -> Result, std::convert::Infallible> { - use hyper::{Body, Response}; - use prometheus::{Encoder, TextEncoder}; - - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - - let metric_families = prometheus::gather(); - encoder - .encode(&metric_families, &mut buffer) - .unwrap_or_log(); - - Ok(Response::new(Body::from(buffer))) -} - -async fn create_metrics_server() { - use hyper::{ - service::{make_service_fn, service_fn}, - Server, - }; - use std::convert::Infallible; - use std::net::SocketAddr; - - let make_svc = - make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(provide_metrics)) }); - - let addr: SocketAddr = std::env::var("METRICS_HOST") - .expect_or_log("Missing METRICS_HOST") - .parse() - .expect_or_log("Invalid METRICS_HOST"); - - let server = Server::bind(&addr).serve(make_svc); - - tokio::spawn(async move { server.await.expect_or_log("Metrics server error") }); -} diff --git a/fuzzysearch-ingest-furaffinity/Cargo.toml b/fuzzysearch-ingest-furaffinity/Cargo.toml index febd753..5dd3855 100644 --- a/fuzzysearch-ingest-furaffinity/Cargo.toml +++ b/fuzzysearch-ingest-furaffinity/Cargo.toml @@ -6,11 +6,8 @@ edition = "2018" [dependencies] reqwest = "0.11" -postgres = { version = "0.19", features = ["with-chrono-0_4"] } tokio = { version = "1", features = ["full"] } -tokio-postgres = { version = "0.7.0" } -r2d2_postgres = " 0.18.0" -r2d2 = "0.8" +tokio-postgres = { version = "0.7.0", features = ["with-chrono-0_4"] } chrono = "0.4" hyper = { version = "0.14", features = ["server"] } prometheus = { version = "0.12", features = ["process"] } @@ -26,4 +23,5 @@ fuzzysearch-common = { path = "../fuzzysearch-common", features = ["queue"] } [dependencies.furaffinity-rs] git = "https://github.com/Syfaro/furaffinity-rs" +branch = "main" features = ["cloudflare-bypass"] diff --git a/fuzzysearch-ingest-furaffinity/Dockerfile b/fuzzysearch-ingest-furaffinity/Dockerfile index 3f9e2cf..2dba3d6 100644 --- a/fuzzysearch-ingest-furaffinity/Dockerfile +++ b/fuzzysearch-ingest-furaffinity/Dockerfile @@ -1,14 +1,7 @@ -FROM rust:1-slim-buster AS builder -WORKDIR /src -ENV SQLX_OFFLINE=true -RUN apt-get update -y && apt-get install -y libssl-dev pkg-config python3 python3-pip -COPY . . -RUN cargo install --root / --path ./fuzzysearch-ingest-furaffinity - FROM debian:buster-slim RUN apt-get update -y && \ apt-get install -y openssl ca-certificates python3 python3-pip && \ python3 -m pip --no-cache-dir install cfscrape && \ rm -rf /var/lib/apt/lists/* -COPY --from=builder /bin/fuzzysearch-ingest-furaffinity /bin/fuzzysearch-ingest-furaffinity +COPY ./fuzzysearch-ingest-furaffinity/fuzzysearch-ingest-furaffinity /bin/fuzzysearch-ingest-furaffinity CMD ["/bin/fuzzysearch-ingest-furaffinity"] diff --git a/fuzzysearch-ingest-furaffinity/src/main.rs b/fuzzysearch-ingest-furaffinity/src/main.rs index 44f1a2d..f04879b 100644 --- a/fuzzysearch-ingest-furaffinity/src/main.rs +++ b/fuzzysearch-ingest-furaffinity/src/main.rs @@ -115,51 +115,6 @@ async fn insert_null_submission(client: &Client, id: i32) -> Result, -) -> Result, hyper::Error> { - match (req.method(), req.uri().path()) { - (&hyper::Method::GET, "/health") => Ok(hyper::Response::new(hyper::Body::from("OK"))), - - (&hyper::Method::GET, "/metrics") => { - use prometheus::Encoder; - - let encoder = prometheus::TextEncoder::new(); - - let metric_families = prometheus::gather(); - let mut buffer = vec![]; - encoder - .encode(&metric_families, &mut buffer) - .unwrap_or_log(); - - Ok(hyper::Response::new(hyper::Body::from(buffer))) - } - - _ => { - let mut not_found = hyper::Response::default(); - *not_found.status_mut() = hyper::StatusCode::NOT_FOUND; - Ok(not_found) - } - } -} - -async fn web() { - use hyper::service::{make_service_fn, service_fn}; - - let addr: std::net::SocketAddr = std::env::var("HTTP_HOST") - .expect_or_log("Missing HTTP_HOST") - .parse() - .unwrap_or_log(); - - let service = make_service_fn(|_| async { Ok::<_, hyper::Error>(service_fn(request)) }); - - let server = hyper::Server::bind(&addr).serve(service); - - tracing::info!("Listening on http://{}", addr); - - server.await.unwrap_or_log(); -} - struct RetryHandler { max_attempts: usize, } @@ -270,7 +225,8 @@ async fn process_submission( #[tokio::main] async fn main() { - fuzzysearch_common::init_logger(); + fuzzysearch_common::trace::configure_tracing("fuzzysearch-ingest-furaffinity"); + fuzzysearch_common::trace::serve_metrics().await; let (cookie_a, cookie_b) = ( std::env::var("FA_A").expect_or_log("Missing FA_A"), @@ -297,8 +253,6 @@ async fn main() { } }); - tokio::spawn(async move { web().await }); - let faktory_dsn = std::env::var("FAKTORY_URL").expect_or_log("Missing FAKTORY_URL"); let faktory = FaktoryClient::connect(faktory_dsn) .await diff --git a/fuzzysearch-ingest-weasyl/Dockerfile b/fuzzysearch-ingest-weasyl/Dockerfile index f2d4556..ed37e4b 100644 --- a/fuzzysearch-ingest-weasyl/Dockerfile +++ b/fuzzysearch-ingest-weasyl/Dockerfile @@ -1,11 +1,4 @@ -FROM rust:1-slim-buster AS builder -WORKDIR /src -ENV SQLX_OFFLINE=true -RUN apt-get update -y && apt-get install -y libssl-dev pkg-config -COPY . . -RUN cargo install --root / --path ./fuzzysearch-ingest-weasyl - FROM debian:buster-slim RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* -COPY --from=builder /bin/fuzzysearch-ingest-weasyl /bin/fuzzysearch-ingest-weasyl +COPY ./fuzzysearch-ingest-weasyl/fuzzysearch-ingest-weasyl /bin/fuzzysearch-ingest-weasyl CMD ["/bin/fuzzysearch-ingest-weasyl"] diff --git a/fuzzysearch-ingest-weasyl/src/main.rs b/fuzzysearch-ingest-weasyl/src/main.rs index 4c5f821..05b52f6 100644 --- a/fuzzysearch-ingest-weasyl/src/main.rs +++ b/fuzzysearch-ingest-weasyl/src/main.rs @@ -198,7 +198,8 @@ async fn insert_null( #[tokio::main] async fn main() { - fuzzysearch_common::init_logger(); + fuzzysearch_common::trace::configure_tracing("fuzzysearch-ingest-weasyl"); + fuzzysearch_common::trace::serve_metrics().await; let api_key = std::env::var("WEASYL_APIKEY").unwrap_or_log(); diff --git a/fuzzysearch-webhook/Dockerfile b/fuzzysearch-webhook/Dockerfile index f760037..f9a111e 100644 --- a/fuzzysearch-webhook/Dockerfile +++ b/fuzzysearch-webhook/Dockerfile @@ -1,12 +1,4 @@ -FROM rust:1-slim-buster AS builder -WORKDIR /src -ENV SQLX_OFFLINE=true -RUN apt-get update -y && apt-get install -y libssl-dev pkg-config -COPY . . -RUN cargo install --root / --path ./fuzzysearch-webhook - FROM debian:buster-slim -WORKDIR /app RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* -COPY --from=builder /bin/fuzzysearch-webhook /bin/fuzzysearch-webhook +COPY ./fuzzysearch-webhook/fuzzysearch-webhook /bin/fuzzysearch-webhook CMD ["/bin/fuzzysearch-webhook"] diff --git a/fuzzysearch-webhook/src/main.rs b/fuzzysearch-webhook/src/main.rs index 43851e9..4493fdc 100644 --- a/fuzzysearch-webhook/src/main.rs +++ b/fuzzysearch-webhook/src/main.rs @@ -27,7 +27,7 @@ pub enum WebhookError { } fn main() { - fuzzysearch_common::init_logger(); + fuzzysearch_common::trace::configure_tracing("fuzzysearch-webhook"); tracing::info!("Starting..."); diff --git a/fuzzysearch/Cargo.toml b/fuzzysearch/Cargo.toml index bb0c93c..8b2c906 100644 --- a/fuzzysearch/Cargo.toml +++ b/fuzzysearch/Cargo.toml @@ -8,17 +8,18 @@ edition = "2018" tracing = "0.1" tracing-subscriber = "0.2" tracing-futures = "0.2" +tracing-log = "0.1" prometheus = { version = "0.12", features = ["process"] } lazy_static = "1" -opentelemetry = { version = "0.14.0", features = ["rt-tokio"] } -opentelemetry-jaeger = { version = "0.13", features = ["tokio"] } -tracing-opentelemetry = "0.12" -opentelemetry-http = "0.3" +opentelemetry = { version = "0.15.0", features = ["rt-tokio"] } +opentelemetry-jaeger = { version = "0.14", features = ["tokio"] } +tracing-opentelemetry = "0.14" +opentelemetry-http = "0.4" tokio = { version = "1", features = ["full"] } -async-stream = "0.3" +tokio-stream = "0.1" futures = "0.3" @@ -29,17 +30,16 @@ serde = { version = "1", features = ["derive"] } serde_json = "1" warp = "0.3" -reqwest = "0.11" +reqwest = { version = "0.11", features = ["multipart"] } hyper = "0.14" sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "macros", "json", "offline"] } -infer = { version = "0.4", default-features = false } ffmpeg-next = "4" image = "0.23" img_hash = "3" hamming = "0.1" -bk-tree = "0.3" +bkapi-client = { git = "https://github.com/Syfaro/bkapi.git" } -fuzzysearch-common = { path = "../fuzzysearch-common", features = ["video"] } +fuzzysearch-common = { path = "../fuzzysearch-common" } diff --git a/fuzzysearch/Dockerfile b/fuzzysearch/Dockerfile index 3acf483..5ede410 100644 --- a/fuzzysearch/Dockerfile +++ b/fuzzysearch/Dockerfile @@ -1,14 +1,6 @@ -FROM rust:1-slim-buster AS builder -WORKDIR /src -ENV SQLX_OFFLINE=true -RUN apt-get update -y && apt-get install -y libssl-dev pkg-config libavcodec-dev libavformat-dev libavutil-dev libavdevice-dev clang llvm -COPY . . -RUN cargo install --root / --path ./fuzzysearch - FROM debian:buster-slim EXPOSE 8080 8081 ENV METRICS_HOST=0.0.0.0:8081 -WORKDIR /app RUN apt-get update -y && apt-get install -y --no-install-recommends openssl ca-certificates ffmpeg && rm -rf /var/lib/apt/lists/* -COPY --from=builder /bin/fuzzysearch /bin/fuzzysearch +COPY ./fuzzysearch/fuzzysearch /bin/fuzzysearch CMD ["/bin/fuzzysearch"] diff --git a/fuzzysearch/src/filters.rs b/fuzzysearch/src/filters.rs index f2e7dd4..c26e39a 100644 --- a/fuzzysearch/src/filters.rs +++ b/fuzzysearch/src/filters.rs @@ -1,19 +1,19 @@ -use crate::types::*; -use crate::{handlers, Pool, Tree}; +use crate::{handlers, Pool}; +use crate::{types::*, Endpoints}; use std::convert::Infallible; use tracing_futures::Instrument; use warp::{Filter, Rejection, Reply}; pub fn search( db: Pool, - tree: Tree, + bkapi: bkapi_client::BKApiClient, + endpoints: Endpoints, ) -> impl Filter + Clone { - search_image(db.clone(), tree.clone()) - .or(search_hashes(db.clone(), tree.clone())) + search_image(db.clone(), bkapi.clone(), endpoints) + .or(search_hashes(db.clone(), bkapi.clone())) .or(search_file(db.clone())) - .or(search_video(db.clone())) .or(check_handle(db.clone())) - .or(search_image_by_url(db, tree)) + .or(search_image_by_url(db, bkapi)) } pub fn search_file(db: Pool) -> impl Filter + Clone { @@ -34,7 +34,8 @@ pub fn search_file(db: Pool) -> impl Filter impl Filter + Clone { warp::path("image") .and(warp::header::headers_cloned()) @@ -42,65 +43,51 @@ pub fn search_image( .and(warp::multipart::form().max_length(1024 * 1024 * 10)) .and(warp::query::()) .and(with_pool(db)) - .and(with_tree(tree)) + .and(with_bkapi(bkapi)) .and(with_api_key()) - .and_then(|headers, form, opts, pool, tree, api_key| { + .and(with_endpoints(endpoints)) + .and_then(|headers, form, opts, pool, bkapi, api_key, endpoints| { use tracing_opentelemetry::OpenTelemetrySpanExt; let span = tracing::info_span!("search_image", ?opts); span.set_parent(with_telem(headers)); span.in_scope(|| { - handlers::search_image(form, opts, pool, tree, api_key).in_current_span() + handlers::search_image(form, opts, pool, bkapi, api_key, endpoints) + .in_current_span() }) }) } pub fn search_image_by_url( db: Pool, - tree: Tree, + bkapi: bkapi_client::BKApiClient, ) -> impl Filter + Clone { warp::path("url") .and(warp::get()) .and(warp::query::()) .and(with_pool(db)) - .and(with_tree(tree)) + .and(with_bkapi(bkapi)) .and(with_api_key()) .and_then(handlers::search_image_by_url) } pub fn search_hashes( db: Pool, - tree: Tree, + bkapi: bkapi_client::BKApiClient, ) -> impl Filter + Clone { warp::path("hashes") .and(warp::header::headers_cloned()) .and(warp::get()) .and(warp::query::()) .and(with_pool(db)) - .and(with_tree(tree)) + .and(with_bkapi(bkapi)) .and(with_api_key()) - .and_then(|headers, opts, db, tree, api_key| { + .and_then(|headers, opts, db, bkapi, api_key| { use tracing_opentelemetry::OpenTelemetrySpanExt; let span = tracing::info_span!("search_hashes", ?opts); span.set_parent(with_telem(headers)); - span.in_scope(|| handlers::search_hashes(opts, db, tree, api_key).in_current_span()) - }) -} - -pub fn search_video(db: Pool) -> impl Filter + Clone { - warp::path("video") - .and(warp::header::headers_cloned()) - .and(warp::post()) - .and(warp::multipart::form().max_length(1024 * 1024 * 10)) - .and(with_pool(db)) - .and(with_api_key()) - .and_then(|headers, form, db, api_key| { - use tracing_opentelemetry::OpenTelemetrySpanExt; - - let span = tracing::info_span!("search_video"); - span.set_parent(with_telem(headers)); - span.in_scope(|| handlers::search_video(form, db, api_key).in_current_span()) + span.in_scope(|| handlers::search_hashes(opts, db, bkapi, api_key).in_current_span()) }) } @@ -120,8 +107,16 @@ fn with_pool(db: Pool) -> impl Filter + C warp::any().map(move || db.clone()) } -fn with_tree(tree: Tree) -> impl Filter + Clone { - warp::any().map(move || tree.clone()) +fn with_bkapi( + bkapi: bkapi_client::BKApiClient, +) -> impl Filter + Clone { + warp::any().map(move || bkapi.clone()) +} + +fn with_endpoints( + endpoints: Endpoints, +) -> impl Filter + Clone { + warp::any().map(move || endpoints.clone()) } fn with_telem(headers: warp::http::HeaderMap) -> opentelemetry::Context { diff --git a/fuzzysearch/src/handlers.rs b/fuzzysearch/src/handlers.rs index a67943a..09b85bc 100644 --- a/fuzzysearch/src/handlers.rs +++ b/fuzzysearch/src/handlers.rs @@ -1,3 +1,6 @@ +use futures::StreamExt; +use futures::TryStreamExt; +use hyper::StatusCode; use lazy_static::lazy_static; use prometheus::{register_histogram, register_int_counter, Histogram, IntCounter}; use std::convert::TryInto; @@ -7,8 +10,12 @@ use warp::{Rejection, Reply}; use crate::models::image_query; use crate::types::*; -use crate::{early_return, rate_limit, Pool, Tree}; -use fuzzysearch_common::types::{SearchResult, SiteInfo}; +use crate::Endpoints; +use crate::{early_return, rate_limit, Pool}; +use fuzzysearch_common::{ + trace::InjectContext, + types::{SearchResult, SiteInfo}, +}; lazy_static! { static ref IMAGE_HASH_DURATION: Histogram = register_histogram!( @@ -37,6 +44,7 @@ lazy_static! { enum Error { Postgres(sqlx::Error), Reqwest(reqwest::Error), + Warp(warp::Error), InvalidData, InvalidImage, ApiKey, @@ -46,7 +54,7 @@ enum Error { impl warp::Reply for Error { fn into_response(self) -> warp::reply::Response { let msg = match self { - Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { + Error::Postgres(_) | Error::Reqwest(_) | Error::Warp(_) => ErrorMessage { code: 500, message: "Internal server error".to_string(), }, @@ -89,98 +97,89 @@ impl From for Error { } } -async fn get_field_bytes(form: warp::multipart::FormData, field: &str) -> bytes::BytesMut { - use bytes::BufMut; - use futures::StreamExt; - - let parts: Vec<_> = form.collect().await; - let mut parts = parts - .into_iter() - .map(|part| { - let part = part.unwrap(); - (part.name().to_string(), part) - }) - .collect::>(); - let data = parts.remove(field).unwrap(); - - data.stream() - .fold(bytes::BytesMut::new(), |mut b, data| { - b.put(data.unwrap()); - async move { b } - }) - .await +impl From for Error { + fn from(err: warp::Error) -> Self { + Self::Warp(err) + } } -#[tracing::instrument(skip(form))] -async fn hash_input(form: warp::multipart::FormData) -> i64 { - let bytes = get_field_bytes(form, "image").await; +#[tracing::instrument(skip(endpoints, form))] +async fn hash_input( + endpoints: &Endpoints, + mut form: warp::multipart::FormData, +) -> Result { + let mut image_part = None; - let len = bytes.len(); - - let _timer = IMAGE_HASH_DURATION.start_timer(); - let hash = tokio::task::spawn_blocking(move || { - let hasher = fuzzysearch_common::get_hasher(); - let image = image::load_from_memory(&bytes).unwrap(); - hasher.hash_image(&image) - }) - .instrument(span!(tracing::Level::TRACE, "hashing image", len)) - .await - .unwrap(); - drop(_timer); - - let mut buf: [u8; 8] = [0; 8]; - buf.copy_from_slice(&hash.as_bytes()); - - i64::from_be_bytes(buf) -} - -#[tracing::instrument(skip(form))] -async fn hash_video(form: warp::multipart::FormData) -> Option> { - use bytes::Buf; - - let bytes = get_field_bytes(form, "video").await; - - let _timer = VIDEO_HASH_DURATION.start_timer(); - let hashes = tokio::task::spawn_blocking(move || { - if infer::is_video(&bytes) { - fuzzysearch_common::video::extract_video_hashes(bytes.reader()).ok() - } else if infer::image::is_gif(&bytes) { - fuzzysearch_common::video::extract_gif_hashes(bytes.reader()).ok() - } else { - None + tracing::debug!("looking at image parts"); + while let Ok(Some(part)) = form.try_next().await { + if part.name() == "image" { + image_part = Some(part); } - }) - .instrument(span!(tracing::Level::TRACE, "hashing video")) - .await - .unwrap(); - drop(_timer); + } - hashes + let image_part = image_part.ok_or(Error::InvalidImage)?; + + tracing::debug!("found image part, reading data"); + let bytes = image_part + .stream() + .fold(bytes::BytesMut::new(), |mut buf, chunk| { + use bytes::BufMut; + + buf.put(chunk.unwrap()); + async move { buf } + }) + .await; + let part = reqwest::multipart::Part::bytes(bytes.to_vec()); + + let form = reqwest::multipart::Form::new().part("image", part); + + tracing::debug!("sending image to hash input service"); + let client = reqwest::Client::new(); + let resp = client + .post(&endpoints.hash_input) + .inject_context() + .multipart(form) + .send() + .await?; + + tracing::debug!("got response"); + if resp.status() != StatusCode::OK { + return Err(Error::InvalidImage); + } + + let hash: i64 = resp + .text() + .await? + .parse() + .map_err(|_err| Error::InvalidImage)?; + + Ok(hash) } pub async fn search_image( form: warp::multipart::FormData, opts: ImageSearchOpts, db: Pool, - tree: Tree, + bkapi: bkapi_client::BKApiClient, api_key: String, + endpoints: Endpoints, ) -> Result, Rejection> { let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); - let num = hash_input(form).await; + let num = early_return!(hash_input(&endpoints, form).await); let mut items = { if opts.search_type == Some(ImageSearchType::Force) { - image_query(db.clone(), tree.clone(), vec![num], 10) + image_query(db.clone(), bkapi.clone(), vec![num], 10) .await .unwrap() } else { - let results = image_query(db.clone(), tree.clone(), vec![num], 0) + let results = image_query(db.clone(), bkapi.clone(), vec![num], 0) .await .unwrap(); if results.is_empty() && opts.search_type != Some(ImageSearchType::Exact) { - image_query(db.clone(), tree.clone(), vec![num], 10) + image_query(db.clone(), bkapi.clone(), vec![num], 10) .await .unwrap() } else { @@ -220,7 +219,7 @@ pub async fn search_image( pub async fn search_hashes( opts: HashSearchOpts, db: Pool, - tree: Tree, + bkapi: bkapi_client::BKApiClient, api_key: String, ) -> Result, Rejection> { let pool = db.clone(); @@ -239,7 +238,7 @@ pub async fn search_hashes( let image_remaining = rate_limit!(&api_key, &db, image_limit, "image", hashes.len() as i16); let results = - early_return!(image_query(pool, tree, hashes.clone(), opts.distance.unwrap_or(10),).await); + early_return!(image_query(pool, bkapi, hashes.clone(), opts.distance.unwrap_or(10)).await); let resp = warp::http::Response::builder() .header("x-rate-limit-total-image", image_remaining.1.to_string()) @@ -385,16 +384,6 @@ pub async fn search_file( Ok(Box::new(resp)) } -pub async fn search_video( - form: warp::multipart::FormData, - _db: Pool, - _api_key: String, -) -> Result { - let hashes = hash_video(form).await; - - Ok(warp::reply::json(&hashes)) -} - pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, Rejection> { let exists = if let Some(handle) = opts.twitter { let result = sqlx::query_scalar!("SELECT exists(SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1))", handle) @@ -413,7 +402,7 @@ pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, pub async fn search_image_by_url( opts: UrlSearchOpts, db: Pool, - tree: Tree, + bkapi: bkapi_client::BKApiClient, api_key: String, ) -> Result, Rejection> { use bytes::BufMut; @@ -470,7 +459,7 @@ pub async fn search_image_by_url( let hash: [u8; 8] = hash.as_bytes().try_into().unwrap(); let num = i64::from_be_bytes(hash); - let results = image_query(db.clone(), tree.clone(), vec![num], 3) + let results = image_query(db.clone(), bkapi.clone(), vec![num], 3) .await .unwrap(); diff --git a/fuzzysearch/src/main.rs b/fuzzysearch/src/main.rs index f6fba39..e177b16 100644 --- a/fuzzysearch/src/main.rs +++ b/fuzzysearch/src/main.rs @@ -1,7 +1,5 @@ #![recursion_limit = "256"] -use std::sync::Arc; -use tokio::sync::RwLock; use warp::Filter; mod filters; @@ -10,37 +8,18 @@ mod models; mod types; mod utils; -type Tree = Arc>>; type Pool = sqlx::PgPool; -#[derive(Debug)] -pub struct Node(pub [u8; 8]); - -impl Node { - pub fn new(hash: i64) -> Self { - Self(hash.to_be_bytes()) - } - - pub fn query(hash: [u8; 8]) -> Self { - Self(hash) - } - - pub fn num(&self) -> i64 { - i64::from_be_bytes(self.0) - } -} - -pub struct Hamming; - -impl bk_tree::Metric for Hamming { - fn distance(&self, a: &Node, b: &Node) -> u64 { - hamming::distance_fast(&a.0, &b.0).unwrap() - } +#[derive(Clone)] +pub struct Endpoints { + pub hash_input: String, + pub bkapi: String, } #[tokio::main] async fn main() { - configure_tracing(); + fuzzysearch_common::trace::configure_tracing("fuzzysearch"); + fuzzysearch_common::trace::serve_metrics().await; ffmpeg_next::init().expect("Unable to initialize ffmpeg"); @@ -50,11 +29,12 @@ async fn main() { .await .expect("Unable to create Postgres pool"); - serve_metrics().await; + let endpoints = Endpoints { + hash_input: std::env::var("ENDPOINT_HASH_INPUT").expect("Missing ENDPOINT_HASH_INPUT"), + bkapi: std::env::var("ENDPOINT_BKAPI").expect("Missing ENDPOINT_BKAPI"), + }; - let tree: Tree = Arc::new(RwLock::new(bk_tree::BKTree::new(Hamming))); - - load_updates(db_pool.clone(), tree.clone()).await; + let bkapi = bkapi_client::BKApiClient::new(&endpoints.bkapi); let log = warp::log("fuzzysearch"); let cors = warp::cors() @@ -64,7 +44,7 @@ async fn main() { let options = warp::options().map(|| "✓"); - let api = options.or(filters::search(db_pool, tree)); + let api = options.or(filters::search(db_pool, bkapi, endpoints)); let routes = api .or(warp::path::end() .map(|| warp::redirect(warp::http::Uri::from_static("https://fuzzysearch.net")))) @@ -74,173 +54,3 @@ async fn main() { warp::serve(routes).run(([0, 0, 0, 0], 8080)).await; } - -fn configure_tracing() { - use opentelemetry::KeyValue; - use tracing_subscriber::layer::SubscriberExt; - - let env = std::env::var("ENVIRONMENT"); - let env = if let Ok(env) = env.as_ref() { - env.as_str() - } else if cfg!(debug_assertions) { - "debug" - } else { - "release" - }; - - opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new()); - - let tracer = opentelemetry_jaeger::new_pipeline() - .with_agent_endpoint(std::env::var("JAEGER_COLLECTOR").expect("Missing JAEGER_COLLECTOR")) - .with_service_name("fuzzysearch") - .with_tags(vec![ - KeyValue::new("environment", env.to_owned()), - KeyValue::new("version", env!("CARGO_PKG_VERSION")), - ]) - .install_batch(opentelemetry::runtime::Tokio) - .unwrap(); - - let trace = tracing_opentelemetry::layer().with_tracer(tracer); - let env_filter = tracing_subscriber::EnvFilter::from_default_env(); - - if matches!(std::env::var("LOG_FMT").as_deref(), Ok("json")) { - let subscriber = tracing_subscriber::fmt::layer() - .json() - .with_timer(tracing_subscriber::fmt::time::ChronoUtc::rfc3339()) - .with_target(true); - let subscriber = tracing_subscriber::Registry::default() - .with(env_filter) - .with(trace) - .with(subscriber); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } else { - let subscriber = tracing_subscriber::fmt::layer(); - let subscriber = tracing_subscriber::Registry::default() - .with(env_filter) - .with(trace) - .with(subscriber); - tracing::subscriber::set_global_default(subscriber).unwrap(); - } -} - -async fn metrics( - _: hyper::Request, -) -> Result, std::convert::Infallible> { - use hyper::{Body, Response}; - use prometheus::{Encoder, TextEncoder}; - - let mut buffer = Vec::new(); - let encoder = TextEncoder::new(); - - let metric_families = prometheus::gather(); - encoder.encode(&metric_families, &mut buffer).unwrap(); - - Ok(Response::new(Body::from(buffer))) -} - -async fn serve_metrics() { - use hyper::{ - service::{make_service_fn, service_fn}, - Server, - }; - use std::convert::Infallible; - use std::net::SocketAddr; - - let make_svc = make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(metrics)) }); - - let addr: SocketAddr = std::env::var("METRICS_HOST") - .expect("Missing METRICS_HOST") - .parse() - .expect("Invalid METRICS_HOST"); - - let server = Server::bind(&addr).serve(make_svc); - - tokio::spawn(async move { - server.await.expect("Metrics server error"); - }); -} - -#[derive(serde::Deserialize)] -struct HashRow { - hash: i64, -} - -async fn create_tree(conn: &Pool) -> bk_tree::BKTree { - use futures::TryStreamExt; - - let mut tree = bk_tree::BKTree::new(Hamming); - - let mut rows = sqlx::query!( - "SELECT hash_int hash FROM submission WHERE hash_int IS NOT NULL - UNION ALL - SELECT hash FROM e621 WHERE hash IS NOT NULL - UNION ALL - SELECT hash FROM tweet_media WHERE hash IS NOT NULL - UNION ALL - SELECT hash FROM weasyl WHERE hash IS NOT NULL" - ) - .fetch(conn); - - let mut count = 0; - - while let Some(row) = rows.try_next().await.expect("Unable to get row") { - if let Some(hash) = row.hash { - let node = Node::new(hash); - if tree.find_exact(&node).is_none() { - tree.add(node); - } - - count += 1; - if count % 250_000 == 0 { - tracing::debug!(count, "Made progress in loading tree rows"); - } - } - } - - tracing::info!(count, "Completed loading rows for tree"); - - tree -} - -async fn load_updates(conn: Pool, tree: Tree) { - let mut listener = sqlx::postgres::PgListener::connect_with(&conn) - .await - .unwrap(); - listener.listen("fuzzysearch_hash_added").await.unwrap(); - - let new_tree = create_tree(&conn).await; - let mut lock = tree.write().await; - *lock = new_tree; - drop(lock); - - tokio::spawn(async move { - loop { - while let Some(notification) = listener - .try_recv() - .await - .expect("Unable to recv notification") - { - let payload: HashRow = serde_json::from_str(notification.payload()).unwrap(); - tracing::debug!(hash = payload.hash, "Adding new hash to tree"); - - let lock = tree.read().await; - if lock.find_exact(&Node::new(payload.hash)).is_some() { - continue; - } - drop(lock); - - let mut lock = tree.write().await; - lock.add(Node(payload.hash.to_be_bytes())); - drop(lock); - } - - tracing::error!("Lost connection to Postgres, recreating tree"); - tokio::time::sleep(std::time::Duration::from_secs(10)).await; - let new_tree = create_tree(&conn).await; - let mut lock = tree.write().await; - *lock = new_tree; - drop(lock); - tracing::info!("Replaced tree"); - } - }); -} diff --git a/fuzzysearch/src/models.rs b/fuzzysearch/src/models.rs index 106690b..7760607 100644 --- a/fuzzysearch/src/models.rs +++ b/fuzzysearch/src/models.rs @@ -2,15 +2,10 @@ use lazy_static::lazy_static; use prometheus::{register_histogram, Histogram}; use crate::types::*; -use crate::{Pool, Tree}; +use crate::Pool; use fuzzysearch_common::types::{SearchResult, SiteInfo}; lazy_static! { - static ref IMAGE_TREE_DURATION: Histogram = register_histogram!( - "fuzzysearch_api_image_tree_seconds", - "Duration to search for hashes in tree" - ) - .unwrap(); static ref IMAGE_QUERY_DURATION: Histogram = register_histogram!( "fuzzysearch_api_image_query_seconds", "Duration to perform a single image lookup query" @@ -51,28 +46,30 @@ struct HashSearch { distance: u64, } -#[tracing::instrument(skip(pool, tree))] +#[tracing::instrument(skip(pool, bkapi))] pub async fn image_query( pool: Pool, - tree: Tree, + bkapi: bkapi_client::BKApiClient, hashes: Vec, distance: i64, ) -> Result, sqlx::Error> { - let timer = IMAGE_TREE_DURATION.start_timer(); - let lock = tree.read().await; - let found_hashes: Vec = hashes - .iter() - .flat_map(|hash| { - lock.find(&crate::Node::new(*hash), distance as u64) - .map(|(dist, node)| HashSearch { - searched_hash: *hash, - found_hash: node.num(), - distance: dist, + let found_hashes: Vec = bkapi + .search_many(&hashes, distance as u64) + .await + .unwrap() + .into_iter() + .flat_map(|results| { + results + .hashes + .iter() + .map(|hash| HashSearch { + searched_hash: results.hash, + found_hash: hash.hash, + distance: hash.distance, }) .collect::>() }) .collect(); - timer.stop_and_record(); let timer = IMAGE_QUERY_DURATION.start_timer(); let matches = sqlx::query!(