From 3ade5aeba94539432c949f96255869a03abf7948 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Thu, 21 Jan 2021 21:21:16 -0500 Subject: [PATCH 1/6] Add ability to search by URL, rate limit headers (#2) * Initial progress on searching by URL. * Avoid rejections for error messages. * Handle some more rejections. * Fix build issues. * Remove detailed error messages. * Fix issue with built Docker image. * Add rate limit headers to all responses. * Remove unneeded dependency. * Limit URLs to 10MB. --- .drone.yml | 4 + Cargo.lock | 1073 +++++++++++++++++++++++++---------------------- Cargo.toml | 19 +- Dockerfile | 2 + src/filters.rs | 18 +- src/handlers.rs | 319 +++++++++----- src/main.rs | 12 +- src/types.rs | 7 +- src/utils.rs | 51 ++- 9 files changed, 878 insertions(+), 627 deletions(-) diff --git a/.drone.yml b/.drone.yml index ba1f691..a723c2f 100644 --- a/.drone.yml +++ b/.drone.yml @@ -37,4 +37,8 @@ steps: exclude: - master +--- +kind: signature +hmac: c17d371c096be4b0544aad509cd58fa475e8737ac2c2dd77db5603044e3a5892 + ... diff --git a/Cargo.lock b/Cargo.lock index b00810d..ee5443e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18,7 +18,28 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d52a9bb7ec0cf484c551830a7ce27bd20d67eac647e1befb56b0be4ee39a55d2" dependencies = [ - "winapi 0.3.9", + "winapi", +] + +[[package]] +name = "async-stream" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3670df70cbc01729f901f94c887814b3c68db038aad1329a418bae178bc5295c" +dependencies = [ + "async-stream-impl", + "futures-core", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a3548b8efc9f8e8a5a0a2808c5bd8451a9031b9e5b879a79590304ae928b0a70" +dependencies = [ + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -32,12 +53,6 @@ dependencies = [ "syn", ] -[[package]] -name = "autocfg" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" - [[package]] name = "autocfg" version = "1.0.1" @@ -50,12 +65,6 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" -[[package]] -name = "base64" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3441f0f7b02788e948e47f457ca01f1d7e6d92c693bc132c22b087d3141c03ff" - [[package]] name = "base64" version = "0.13.0" @@ -64,26 +73,26 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bb8" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a4d5a11ea6fe65800f3225ab57f7c28024595c99e809a0ca7eee60a8e3fa24b" +checksum = "dae93eccab998c4b8703e3a6bbaa1714c38e445ebacb4bede25d0408521e293c" dependencies = [ "async-trait", "futures-channel", "futures-util", "parking_lot", - "tokio 0.3.5", + "tokio", ] [[package]] name = "bb8-postgres" -version = "0.6.0" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7556910cdbd798b51b16da3a93dfc4cc303b326503a8d03e830d11fe3969ae1d" +checksum = "61fdf56d52b2cca401d2380407e5c35d3d25d3560224ecf74d6e4ca13e51239b" dependencies = [ "async-trait", "bb8", - "tokio 0.3.5", + "tokio", "tokio-postgres", ] @@ -139,6 +148,12 @@ dependencies = [ "safemem", ] +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + [[package]] name = "byte-tools" version = "0.3.1" @@ -147,27 +162,21 @@ checksum = "e3b5ca7a04898ad4bcd41c90c5285445ff5b791899bb1b0abdd2a2aa791211d7" [[package]] name = "bytemuck" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41aa2ec95ca3b5c54cf73c91acf06d24f4495d5f1b1c12506ae3483d646177ac" +checksum = "5a4bad0c5981acc24bc09e532f35160f952e35422603f0563cd7a73c2c2e65a0" [[package]] name = "byteorder" -version = "1.3.4" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" [[package]] name = "bytes" -version = "0.5.6" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - -[[package]] -name = "bytes" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] name = "cc" @@ -197,16 +206,7 @@ dependencies = [ "num-integer", "num-traits", "time", - "winapi 0.3.9", -] - -[[package]] -name = "cloudabi" -version = "0.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" -dependencies = [ - "bitflags", + "winapi", ] [[package]] @@ -217,9 +217,25 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" [[package]] name = "const_fn" -version = "0.4.3" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" +checksum = "28b9d6de7f49e22cf97ad17fc4036ece69300032f45f78f30b4a4482cdc3f4a6" + +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" [[package]] name = "cpuid-bool" @@ -277,16 +293,16 @@ version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "02d96d1e189ef58269ebe5b97953da3274d83a93af647c2ddd6f9dab28cedb8d" dependencies = [ - "autocfg 1.0.1", + "autocfg", "cfg-if 1.0.0", "lazy_static", ] [[package]] name = "crypto-mac" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58bcd97a54c7ca5ce2f6eb16f6bede5b0ab5f0055fedc17d2f0b4466e21671ca" +checksum = "4857fd85a0c34b3c3297875b747c1e02e06b6a0ea32dd892d8192b9ce0813ea6" dependencies = [ "generic-array 0.14.4", "subtle", @@ -320,18 +336,21 @@ dependencies = [ "generic-array 0.14.4", ] -[[package]] -name = "dtoa" -version = "0.4.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" - [[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -350,6 +369,21 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.0" @@ -360,33 +394,11 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "fuchsia-cprng" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" - -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - [[package]] name = "futures" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b3b0c040a1fe6529d30b3c5944b280c7f0dcb2930d2c3062bca967b602583d0" +checksum = "da9052a1a50244d8d5aa9bf55cbc2fb6f357c86cc52e46c62ed390a7180cf150" dependencies = [ "futures-channel", "futures-core", @@ -399,9 +411,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4b7109687aa4e177ef6fe84553af6280ef2778bdb7783ba44c9dc3399110fe64" +checksum = "f2d31b7ec7efab6eefc7c57233bb10b847986139d88cc2f5a02a1ae6871a1846" dependencies = [ "futures-core", "futures-sink", @@ -409,15 +421,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "847ce131b72ffb13b6109a221da9ad97a64cbe48feb1028356b836b47b8f1748" +checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65" [[package]] name = "futures-executor" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4caa2b2b68b880003057c1dd49f1ed937e38f22fcf6c212188a121f08cf40a65" +checksum = "e9e59fdc009a4b3096bf94f740a0f2424c082521f20a9b08c5c07c48d90fd9b9" dependencies = [ "futures-core", "futures-task", @@ -426,15 +438,15 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "611834ce18aaa1bd13c4b374f5d653e1027cf99b6b502584ff8c9a64413b30bb" +checksum = "28be053525281ad8259d47e4de5de657b25e7bac113458555bb4b70bc6870500" [[package]] name = "futures-macro" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77408a692f1f97bcc61dc001d752e00643408fbc922e4d634c655df50d595556" +checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd" dependencies = [ "proc-macro-hack", "proc-macro2", @@ -444,24 +456,24 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f878195a49cee50e006b02b93cf7e0a95a38ac7b776b4c4d9cc1207cd20fcb3d" +checksum = "caf5c69029bda2e743fddd0582d1083951d65cc9539aebf8812f36c3491342d6" [[package]] name = "futures-task" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c554eb5bf48b2426c4771ab68c6b14468b6e76cc90996f528c3338d761a4d0d" +checksum = "13de07eb8ea81ae445aca7b69f5f7bf15d7bf4912d8ca37d6645c77ae8a58d86" dependencies = [ "once_cell", ] [[package]] name = "futures-util" -version = "0.3.8" +version = "0.3.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d304cff4a7b99cfb7986f7d43fbe93d175e72e704a8860787cc95e9ffd85cbd2" +checksum = "632a8cd0f2a4b3fdea1657f08bde063848c3bd00f9bbf6e256b8be78802e624b" dependencies = [ "futures-channel", "futures-core", @@ -470,7 +482,7 @@ dependencies = [ "futures-sink", "futures-task", "memchr", - "pin-project 1.0.2", + "pin-project-lite", "pin-utils", "proc-macro-hack", "proc-macro-nested", @@ -481,20 +493,23 @@ dependencies = [ name = "fuzzysearch" version = "0.1.0" dependencies = [ + "async-stream", "bb8", "bb8-postgres", "bk-tree", - "bytes 0.5.6", + "bytes", "chrono", "futures", - "futures-util", "hamming", + "hyper", "image", "img_hash", "opentelemetry", "opentelemetry-jaeger", + "reqwest", "serde", - "tokio 0.3.5", + "serde_json", + "tokio", "tokio-postgres", "tracing", "tracing-futures", @@ -503,19 +518,6 @@ dependencies = [ "warp", ] -[[package]] -name = "generator" -version = "0.6.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cdc09201b2e8ca1b19290cf7e65de2246b8e91fb6874279722189c4de7b94dc" -dependencies = [ - "cc", - "libc", - "log", - "rustc_version", - "winapi 0.3.9", -] - [[package]] name = "generic-array" version = "0.12.3" @@ -537,15 +539,26 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.1.15" +version = "0.1.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "getrandom" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" +dependencies = [ + "cfg-if 1.0.0", + "libc", + "wasi 0.10.1+wasi-snapshot-preview1", +] + [[package]] name = "gif" version = "0.11.1" @@ -558,11 +571,11 @@ dependencies = [ [[package]] name = "h2" -version = "0.2.7" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" +checksum = "6b67e66362108efccd8ac053abafc8b7a8d86a37e6e48fc4f6f7485eb5e9e6a5" dependencies = [ - "bytes 0.5.6", + "bytes", "fnv", "futures-core", "futures-sink", @@ -570,8 +583,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 0.2.23", - "tokio-util 0.3.1", + "tokio", + "tokio-util", "tracing", "tracing-futures", ] @@ -590,13 +603,13 @@ checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" [[package]] name = "headers" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" +checksum = "62689dc57c7456e69712607ffcbd0aa1dfcccf9af73727e9b25bc1825375cac3" dependencies = [ - "base64 0.12.3", + "base64 0.13.0", "bitflags", - "bytes 0.5.6", + "bytes", "headers-core", "http", "mime", @@ -615,18 +628,18 @@ dependencies = [ [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] [[package]] name = "hmac" -version = "0.9.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deae6d9dbb35ec2c502d62b8f7b1c000a0822c3b0794ba36b3149c0a1c840dff" +checksum = "c1441c6b1e930e2817404b5046f1f989899143a12bf92de603b69f4e0aee1e15" dependencies = [ "crypto-mac", "digest 0.9.0", @@ -634,22 +647,22 @@ dependencies = [ [[package]] name = "http" -version = "0.2.1" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" +checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" dependencies = [ - "bytes 0.5.6", + "bytes", "fnv", "itoa", ] [[package]] name = "http-body" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" +checksum = "2861bd27ee074e5ee891e8b539837a9430012e249d7f0ca2d795650f579c1994" dependencies = [ - "bytes 0.5.6", + "bytes", "http", ] @@ -667,11 +680,11 @@ checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "hyper" -version = "0.13.9" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" +checksum = "12219dc884514cb4a6a03737f4413c0e01c23a1b059b0156004b23f1e19dccbe" dependencies = [ - "bytes 0.5.6", + "bytes", "futures-channel", "futures-core", "futures-util", @@ -681,14 +694,27 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.2", + "pin-project 1.0.4", "socket2", - "tokio 0.2.23", + "tokio", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper-tls" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" +dependencies = [ + "bytes", + "hyper", + "native-tls", + "tokio", + "tokio-native-tls", +] + [[package]] name = "idna" version = "0.2.0" @@ -734,21 +760,21 @@ dependencies = [ [[package]] name = "indexmap" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55e2e4c765aa53a0424761bf9f41aa7a6ac1efa87238f59560640e27fca028f2" +checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" dependencies = [ - "autocfg 1.0.1", + "autocfg", "hashbrown", ] [[package]] name = "input_buffer" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" +checksum = "f97967975f448f1a7ddb12b0bc41069d09ed6a1c161a92687e057325db35d413" dependencies = [ - "bytes 0.5.6", + "bytes", ] [[package]] @@ -762,24 +788,21 @@ dependencies = [ [[package]] name = "integer-encoding" -version = "1.1.5" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f4ebd0bd29be0f11973e9b3e219005661042a019fd757798c36a47c87852625" +checksum = "f6104619c35f8835695e517cfb80fb7142139ee4b53f4d0fa4c8dca6e98fbc66" [[package]] -name = "iovec" -version = "0.1.4" +name = "ipnet" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", -] +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jpeg-decoder" @@ -792,13 +815,12 @@ dependencies = [ ] [[package]] -name = "kernel32-sys" -version = "0.2.2" +name = "js-sys" +version = "0.3.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" dependencies = [ - "winapi 0.2.8", - "winapi-build", + "wasm-bindgen", ] [[package]] @@ -809,9 +831,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.81" +version = "0.2.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1482821306169ec4d07f6aca392a4681f66c75c9918aa49641a2595db64053cb" +checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" [[package]] name = "lock_api" @@ -824,26 +846,13 @@ dependencies = [ [[package]] name = "log" -version = "0.4.11" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" dependencies = [ "cfg-if 0.1.10", ] -[[package]] -name = "loom" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a0e8460f2f2121162705187214720353c517b97bdfb3494c0b1e33d83ebe4bed" -dependencies = [ - "cfg-if 0.1.10", - "generator", - "scoped-tls", - "serde", - "serde_json", -] - [[package]] name = "matchers" version = "0.0.1" @@ -877,7 +886,7 @@ version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "157b4208e3059a8f9e78d559edc658e13df41410cb3ae03979c83130067fdd87" dependencies = [ - "autocfg 1.0.1", + "autocfg", ] [[package]] @@ -912,51 +921,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" dependencies = [ "adler", - "autocfg 1.0.1", + "autocfg", ] [[package]] name = "mio" -version = "0.6.23" +version = "0.7.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4afd66f5b91bf2a3bc13fad0e21caedac168ca4c707504e75585648ae80e4cc4" -dependencies = [ - "cfg-if 0.1.10", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow 0.2.2", - "net2", - "slab", - "winapi 0.2.8", -] - -[[package]] -name = "mio" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f33bc887064ef1fd66020c9adfc45bb9f33d75a42096c81e7c56c65b75dd1a8b" +checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7" dependencies = [ "libc", "log", - "miow 0.3.6", + "miow", "ntapi", - "winapi 0.3.9", -] - -[[package]] -name = "miow" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebd808424166322d4a38da87083bfddd3ac4c131334ed55856112eb06d46944d" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", + "winapi", ] [[package]] @@ -966,14 +944,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a33c1b55807fbed163481b5ba66db4b2fa6cde694a5027be10fb724206c5897" dependencies = [ "socket2", - "winapi 0.3.9", + "winapi", ] [[package]] name = "multipart" -version = "0.17.0" +version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8209c33c951f07387a8497841122fc6f712165e3f9bda3e6be4645b58188f676" +checksum = "d050aeedc89243f5347c3e237e3e13dc76fbe4ae3742a57b94dc14f69acf76d4" dependencies = [ "buf_redux", "httparse", @@ -981,21 +959,28 @@ dependencies = [ "mime", "mime_guess", "quick-error", - "rand 0.6.5", + "rand 0.7.3", "safemem", "tempfile", "twoway", ] [[package]] -name = "net2" -version = "0.2.36" +name = "native-tls" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7cf75f38f16cb05ea017784dc6dbfd354f76c223dba37701734c4f5a9337d02" +checksum = "b8d96b2e1c8da3957d58100b09f102c6d9cfdfced01b7ec5a8974044bb09dbd4" dependencies = [ - "cfg-if 0.1.10", + "lazy_static", "libc", - "winapi 0.3.9", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", ] [[package]] @@ -1004,7 +989,7 @@ version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1013,7 +998,7 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6b19411a9719e753aff12e5187b74d60d3dc449ec3f4dc21e3989c3f554bc95" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-traits", ] @@ -1023,7 +1008,7 @@ version = "0.1.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2cc698a63b549a70bc047073d2949cce27cd1c7b0a4a862d08a8031bc2801db" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-traits", ] @@ -1033,7 +1018,7 @@ version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2021c8337a54d21aca0d59a92577a029af9431cb59b909b03252b9c164fad59" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-integer", "num-traits", ] @@ -1044,7 +1029,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "12ac428b1cb17fce6f731001d307d351ec70a6d202fc2e60f7d4c5e42d8f4f07" dependencies = [ - "autocfg 1.0.1", + "autocfg", "num-integer", "num-traits", ] @@ -1055,7 +1040,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a64b1ec5cda2586e284722486d802acf1f7dbdc623e2bfc57e65ca1cd099290" dependencies = [ - "autocfg 1.0.1", + "autocfg", ] [[package]] @@ -1086,6 +1071,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038d43985d1ddca7a9900630d8cd031b56e4794eecc2e9ea39dd17aa04399a70" +dependencies = [ + "bitflags", + "cfg-if 1.0.0", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.60" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "921fc71883267538946025deffb622905ecad223c28efbfdef9bb59a0175f3e6" +dependencies = [ + "autocfg", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.6.0" @@ -1132,16 +1150,16 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7c6d9b8427445284a09c55be860a15855ab580a417ccad9da88f5a06787ced0" +checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall", + "redox_syscall 0.1.57", "smallvec", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1179,11 +1197,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccc2237c2c489783abd8c4c80e5450fc0e98644555b1364da68cc29aa151ca7" +checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" dependencies = [ - "pin-project-internal 1.0.2", + "pin-project-internal 1.0.4", ] [[package]] @@ -1199,9 +1217,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.2" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8e8d2bf0b23038a4424865103a4df472855692821aab4e4f5c3312d461d9e5f" +checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" dependencies = [ "proc-macro2", "quote", @@ -1210,15 +1228,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.1.11" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c917123afa01924fc84bb20c4c03f004d9c38e5127e3c039bbf7f4b9c76a2f6b" - -[[package]] -name = "pin-project-lite" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b063f57ec186e6140e2b8b6921e5f1bd89c7356dda5b33acc5401203ca6131c" +checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" [[package]] name = "pin-utils" @@ -1227,10 +1239,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] -name = "png" -version = "0.16.7" +name = "pkg-config" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfe7f9f1c730833200b134370e1d5098964231af8450bce9b78ee3ab5278b970" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "png" +version = "0.16.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c3287920cb847dee3de33d301c463fba14dda99db24214ddf93f83d3021f4c6" dependencies = [ "bitflags", "crc32fast", @@ -1240,29 +1258,29 @@ dependencies = [ [[package]] name = "postgres-protocol" -version = "0.5.3" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4888a0e36637ab38d76cace88c1476937d617ad015f07f6b669cec11beacc019" +checksum = "70e34ad3dc5c56d036b9418185ee97e14b6766d55c8ccf9dc18302ad4e6371d9" dependencies = [ "base64 0.13.0", "byteorder", - "bytes 0.5.6", + "bytes", "fallible-iterator", "hmac", "md5", "memchr", - "rand 0.7.3", + "rand 0.8.2", "sha2", "stringprep", ] [[package]] name = "postgres-types" -version = "0.1.3" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cfc08a7d94a80665de4a83942fa8db2fdeaf2f123fc0535e384dc4fff251efae" +checksum = "5493d9d4613b88b12433aa12890e74e74cd93fdc1e08b7c2aed4768aaae8414c" dependencies = [ - "bytes 0.5.6", + "bytes", "fallible-iterator", "postgres-protocol", ] @@ -1281,9 +1299,9 @@ checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" [[package]] name = "proc-macro-nested" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eba180dafb9038b050a4c280019bbedf9f2467b61e5d892dcad585bb57aadc5a" +checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" [[package]] name = "proc-macro2" @@ -1310,9 +1328,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.18.1" +version = "2.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da78e04bc0e40f36df43ecc6575e4f4b180e8156c4efd73f13d5619479b05696" +checksum = "86473d5f16580f10b131a0bf0afb68f8e029d1835d33a00f37281b05694e5312" [[package]] name = "quick-error" @@ -1322,39 +1340,20 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.7" +version = "1.0.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" dependencies = [ "proc-macro2", ] -[[package]] -name = "rand" -version = "0.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d71dacdc3c88c1fde3885a3be3fbab9f35724e6ce99467f7d9c5026132184ca" -dependencies = [ - "autocfg 0.1.7", - "libc", - "rand_chacha 0.1.1", - "rand_core 0.4.2", - "rand_hc 0.1.0", - "rand_isaac", - "rand_jitter", - "rand_os", - "rand_pcg", - "rand_xorshift", - "winapi 0.3.9", -] - [[package]] name = "rand" version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" dependencies = [ - "getrandom", + "getrandom 0.1.16", "libc", "rand_chacha 0.2.2", "rand_core 0.5.1", @@ -1362,13 +1361,15 @@ dependencies = [ ] [[package]] -name = "rand_chacha" -version = "0.1.1" +name = "rand" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556d3a1ca6600bfcbab7c7c91ccb085ac7fbbcd70e008a98742e7847f4f7bcef" +checksum = "18519b42a40024d661e1714153e9ad0c3de27cd495760ceb09710920f1098b1e" dependencies = [ - "autocfg 0.1.7", - "rand_core 0.3.1", + "libc", + "rand_chacha 0.3.0", + "rand_core 0.6.1", + "rand_hc 0.3.0", ] [[package]] @@ -1382,36 +1383,31 @@ dependencies = [ ] [[package]] -name = "rand_core" -version = "0.3.1" +name = "rand_chacha" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6fdeb83b075e8266dcc8762c22776f6877a63111121f5f8c7411e5be7eed4b" +checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ - "rand_core 0.4.2", + "ppv-lite86", + "rand_core 0.6.1", ] -[[package]] -name = "rand_core" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c33a3c44ca05fa6f1807d8e6743f3824e8509beca625669633be0acbdf509dc" - [[package]] name = "rand_core" version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" dependencies = [ - "getrandom", + "getrandom 0.1.16", ] [[package]] -name = "rand_hc" -version = "0.1.0" +name = "rand_core" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b40677c7be09ae76218dc623efbf7b18e34bced3f38883af07bb75630a21bc4" +checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" dependencies = [ - "rand_core 0.3.1", + "getrandom 0.2.2", ] [[package]] @@ -1424,56 +1420,12 @@ dependencies = [ ] [[package]] -name = "rand_isaac" -version = "0.1.1" +name = "rand_hc" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ded997c9d5f13925be2a6fd7e66bf1872597f759fd9dd93513dd7e92e5a5ee08" +checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" dependencies = [ - "rand_core 0.3.1", -] - -[[package]] -name = "rand_jitter" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1166d5c91dc97b88d1decc3285bb0a99ed84b05cfd0bc2341bdf2d43fc41e39b" -dependencies = [ - "libc", - "rand_core 0.4.2", - "winapi 0.3.9", -] - -[[package]] -name = "rand_os" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b75f676a1e053fc562eafbb47838d67c84801e38fc1ba459e8f180deabd5071" -dependencies = [ - "cloudabi", - "fuchsia-cprng", - "libc", - "rand_core 0.4.2", - "rdrand", - "winapi 0.3.9", -] - -[[package]] -name = "rand_pcg" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf9b09b01790cfe0364f52bf32995ea3c39f4d2dd011eac241d2914146d0b44" -dependencies = [ - "autocfg 0.1.7", - "rand_core 0.4.2", -] - -[[package]] -name = "rand_xorshift" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbf7e9e623549b0e21f6e97cf8ecf247c1a8fd2e8a992ae265314300b2455d5c" -dependencies = [ - "rand_core 0.3.1", + "rand_core 0.6.1", ] [[package]] @@ -1482,7 +1434,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b0d8e0819fadc20c74ea8373106ead0600e3a67ef1fe8da56e39b9ae7275674" dependencies = [ - "autocfg 1.0.1", + "autocfg", "crossbeam-deque", "either", "rayon-core", @@ -1501,15 +1453,6 @@ dependencies = [ "num_cpus", ] -[[package]] -name = "rdrand" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "678054eb77286b51581ba43620cc911abf02758c91f93f479767aed0f90458b2" -dependencies = [ - "rand_core 0.3.1", -] - [[package]] name = "redox_syscall" version = "0.1.57" @@ -1517,10 +1460,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] -name = "regex" -version = "1.4.2" +name = "redox_syscall" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" +checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570" +dependencies = [ + "bitflags", +] + +[[package]] +name = "regex" +version = "1.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ "regex-syntax", ] @@ -1537,9 +1489,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.6.21" +version = "0.6.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b181ba2dcf07aaccad5448e8ead58db5b742cf85dfe035e2227f137a539a189" +checksum = "b5eb417147ba9860a96cfe72a0b93bf88fee1744b5636ec99ab20c1aa9376581" [[package]] name = "remove_dir_all" @@ -1547,16 +1499,41 @@ version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3acd125665422973a33ac9d3dd2df85edad0f4ae9b00dafb1a05e43a9f5ef8e7" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] -name = "rustc_version" -version = "0.2.3" +name = "reqwest" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +checksum = "fd281b1030aa675fb90aa994d07187645bb3c8fc756ca766e7c3070b439de9de" dependencies = [ - "semver", + "base64 0.13.0", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "serde", + "serde_urlencoded", + "tokio", + "tokio-native-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "winreg", ] [[package]] @@ -1593,6 +1570,16 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi", +] + [[package]] name = "scoped-tls" version = "1.0.0" @@ -1612,34 +1599,42 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] -name = "semver" -version = "0.9.0" +name = "security-framework" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" dependencies = [ - "semver-parser", + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", ] [[package]] -name = "semver-parser" -version = "0.7.0" +name = "security-framework-sys" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" +checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" +dependencies = [ + "core-foundation-sys", + "libc", +] [[package]] name = "serde" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c64263859d87aa2eb554587e2d23183398d617427327cf2b3d0ed8c69e4800" +checksum = "166b2349061381baf54a58e4b13c89369feb0ef2eaa57198899e2312aac30aab" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.118" +version = "1.0.120" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c84d3526699cd55261af4b941e4e725444df67aa4f9e6a3564f18030d12672df" +checksum = "0ca2a8cb5805ce9e3b95435e3765b7b553cecc762d938d409434338386cb5775" dependencies = [ "proc-macro2", "quote", @@ -1648,9 +1643,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.60" +version = "1.0.61" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779" +checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" dependencies = [ "itoa", "ryu", @@ -1659,14 +1654,14 @@ dependencies = [ [[package]] name = "serde_urlencoded" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ec5d77e2d4c73717816afac02670d5c4f534ea95ed430442cad02e7a6e32c97" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" dependencies = [ - "dtoa", + "form_urlencoded", "itoa", + "ryu", "serde", - "url", ] [[package]] @@ -1709,12 +1704,20 @@ dependencies = [ [[package]] name = "sharded-slab" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b4921be914e16899a80adefb821f8ddb7974e3f1250223575a44ed994882127" +checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3" dependencies = [ "lazy_static", - "loom", +] + +[[package]] +name = "signal-hook-registry" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16f1d0fef1604ba8f7a073c7e701f213e056707210e9020af4528e0101ce11a6" +dependencies = [ + "libc", ] [[package]] @@ -1731,20 +1734,19 @@ checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" [[package]] name = "smallvec" -version = "1.5.1" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae524f056d7d770e174287294f562e95044c68e88dec909a00d2094805db9d75" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "socket2" -version = "0.3.17" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c29947abdee2a218277abeca306f25789c938e500ea5a9d4b12a5a504466902" +checksum = "122e570113d28d773067fab24266b66753f6ea915758651696b6e35e49f88d6e" dependencies = [ "cfg-if 1.0.0", "libc", - "redox_syscall", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1771,15 +1773,15 @@ dependencies = [ [[package]] name = "subtle" -version = "2.3.0" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "343f3f510c2915908f155e94f17220b19ccfacf2a64a2a5d8004f2c3e311e7fd" +checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.53" +version = "1.0.58" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8833e20724c24de12bbaba5ad230ea61c3eafb05b881c7c9d3cfe8638b187e68" +checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" dependencies = [ "proc-macro2", "quote", @@ -1788,23 +1790,23 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.1.0" +version = "3.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6e24d9338a0a5be79593e2fa15a648add6138caa803e2d5bc782c371732ca9" +checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "libc", - "rand 0.7.3", - "redox_syscall", + "rand 0.8.2", + "redox_syscall 0.2.4", "remove_dir_all", - "winapi 0.3.9", + "winapi", ] [[package]] name = "thread_local" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" +checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447" dependencies = [ "lazy_static", ] @@ -1833,9 +1835,9 @@ dependencies = [ [[package]] name = "tiff" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abeb4e3f32a8973722c0254189e6890358e72b1bf11becb287ee0b23c595a41d" +checksum = "9a53f4706d65497df0c4349241deddf35f84cee19c87ed86ea8ca590f4464437" dependencies = [ "jpeg-decoder", "miniz_oxide 0.4.3", @@ -1844,13 +1846,12 @@ dependencies = [ [[package]] name = "time" -version = "0.1.44" +version = "0.1.43" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" dependencies = [ "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1870,46 +1871,29 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "0.2.23" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff" +checksum = "0ca04cec6ff2474c638057b65798f60ac183e5e79d3448bb7163d36a39cff6ec" dependencies = [ - "bytes 0.5.6", - "fnv", - "futures-core", - "iovec", - "lazy_static", - "memchr", - "mio 0.6.23", - "pin-project-lite 0.1.11", - "slab", -] - -[[package]] -name = "tokio" -version = "0.3.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12a3eb39ee2c231be64487f1fcbe726c8f2514876a55480a5ab8559fc374252" -dependencies = [ - "autocfg 1.0.1", - "bytes 0.6.0", - "futures-core", - "lazy_static", + "autocfg", + "bytes", "libc", "memchr", - "mio 0.7.6", + "mio", "num_cpus", + "once_cell", "parking_lot", - "pin-project-lite 0.2.0", - "slab", + "pin-project-lite", + "signal-hook-registry", "tokio-macros", + "winapi", ] [[package]] name = "tokio-macros" -version = "0.3.1" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "21d30fdbb5dc2d8f91049691aa1a9d4d4ae422a21c334ce8936e5886d30c5c45" +checksum = "42517d2975ca3114b22a16192634e8241dc5cc1f130be194645970cc1c371494" dependencies = [ "proc-macro2", "quote", @@ -1917,66 +1901,76 @@ dependencies = [ ] [[package]] -name = "tokio-postgres" -version = "0.6.0" +name = "tokio-native-tls" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "150d9be163b0df6dc185b8ee33bcb9a74865f0cad754495847f2e06e2051a345" +checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-postgres" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cc9f82c2bfb06a33dd0dfb44b07ca98fe72df19e681d80c78d05a1bac2138e2" dependencies = [ "async-trait", "byteorder", - "bytes 0.5.6", + "bytes", "fallible-iterator", "futures", "log", "parking_lot", "percent-encoding", "phf", - "pin-project-lite 0.1.11", + "pin-project-lite", "postgres-protocol", "postgres-types", - "tokio 0.3.5", - "tokio-util 0.4.0", + "socket2", + "tokio", + "tokio-util", +] + +[[package]] +name = "tokio-stream" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76066865172052eb8796c686f0b441a93df8b08d40a950b062ffb9a426f00edd" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", ] [[package]] name = "tokio-tungstenite" -version = "0.11.0" +version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d9e878ad426ca286e4dcae09cbd4e1973a7f8987d97570e2469703dd7f5720c" +checksum = "e1a5f475f1b9d077ea1017ecbc60890fda8e54942d680ca0b1d2b47cfa2d861b" dependencies = [ "futures-util", "log", - "pin-project 0.4.27", - "tokio 0.2.23", + "pin-project 1.0.4", + "tokio", "tungstenite", ] [[package]] name = "tokio-util" -version = "0.3.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" +checksum = "feb971a26599ffd28066d387f109746df178eff14d5ea1e235015c5601967a4b" dependencies = [ - "bytes 0.5.6", + "async-stream", + "bytes", "futures-core", "futures-sink", "log", - "pin-project-lite 0.1.11", - "tokio 0.2.23", -] - -[[package]] -name = "tokio-util" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24793699f4665ba0416ed287dc794fe6b11a4aa5e4e95b58624f45f6c46b97d4" -dependencies = [ - "bytes 0.5.6", - "futures-core", - "futures-sink", - "log", - "pin-project-lite 0.1.11", - "tokio 0.3.5", + "pin-project-lite", + "tokio", + "tokio-stream", ] [[package]] @@ -1993,7 +1987,7 @@ checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" dependencies = [ "cfg-if 1.0.0", "log", - "pin-project-lite 0.2.0", + "pin-project-lite", "tracing-attributes", "tracing-core", ] @@ -2109,18 +2103,18 @@ checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" [[package]] name = "tungstenite" -version = "0.11.1" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0308d80d86700c5878b9ef6321f020f29b1bb9d5ff3cab25e75e23f3a492a23" +checksum = "8ada8297e8d70872fa9a551d93250a9f407beb9f37ef86494eb20012a2ff7c24" dependencies = [ - "base64 0.12.3", + "base64 0.13.0", "byteorder", - "bytes 0.5.6", + "bytes", "http", "httparse", "input_buffer", "log", - "rand 0.7.3", + "rand 0.8.2", "sha-1 0.9.2", "url", "utf-8", @@ -2186,18 +2180,18 @@ dependencies = [ "percent-encoding", ] -[[package]] -name = "urlencoding" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9232eb53352b4442e40d7900465dfc534e8cb2dc8f18656fcb2ac16112b5593" - [[package]] name = "utf-8" version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05e42f7c18b8f902290b009cde6d651262f956c98bc51bca4cd1d511c9cd85c7" +[[package]] +name = "vcpkg" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb" + [[package]] name = "version_check" version = "0.9.2" @@ -2216,11 +2210,11 @@ dependencies = [ [[package]] name = "warp" -version = "0.2.5" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f41be6df54c97904af01aa23e613d4521eed7ab23537cede692d4058f6449407" +checksum = "3dafd0aac2818a94a34df0df1100a7356c493d8ede4393875fd0b5c51bb6bc80" dependencies = [ - "bytes 0.5.6", + "bytes", "futures", "headers", "http", @@ -2229,17 +2223,19 @@ dependencies = [ "mime", "mime_guess", "multipart", - "pin-project 0.4.27", + "percent-encoding", + "pin-project 1.0.4", "scoped-tls", "serde", "serde_json", "serde_urlencoded", - "tokio 0.2.23", + "tokio", + "tokio-stream", "tokio-tungstenite", + "tokio-util", "tower-service", "tracing", "tracing-futures", - "urlencoding", ] [[package]] @@ -2250,9 +2246,87 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.0+wasi-snapshot-preview1" +version = "0.10.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +checksum = "93c6c3420963c5c64bca373b25e77acb562081b9bb4dd5bb864187742186cea9" + +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] [[package]] name = "weezl" @@ -2260,12 +2334,6 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277" -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" - [[package]] name = "winapi" version = "0.3.9" @@ -2276,12 +2344,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -2295,11 +2357,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] -name = "ws2_32-sys" -version = "0.2.1" +name = "winreg" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ - "winapi 0.2.8", - "winapi-build", + "winapi", ] diff --git a/Cargo.toml b/Cargo.toml index 83a7ce7..02404a6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -13,19 +13,24 @@ opentelemetry = "0.6" opentelemetry-jaeger = "0.5" tracing-opentelemetry = "0.5" -tokio = { version = "0.3", features = ["macros", "rt-multi-thread", "sync"] } +tokio = { version = "1", features = ["full"] } +async-stream = "0.3" + futures = "0.3" -futures-util = "0.3" chrono = "0.4" -bytes = "0.5" +bytes = "1" serde = { version = "1", features = ["derive"] } -warp = "0.2" +serde_json = "1" -tokio-postgres = "0.6" -bb8 = "0.6" -bb8-postgres = "0.6" +warp = "0.3" +reqwest = "0.11" +hyper = "0.14" + +tokio-postgres = "0.7" +bb8 = "0.7" +bb8-postgres = "0.7" img_hash = "3" image = "0.23" diff --git a/Dockerfile b/Dockerfile index 186616d..46e231d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,10 +1,12 @@ FROM rust:1-slim AS builder WORKDIR /src +RUN apt-get update -y && apt-get install -y libssl-dev pkg-config COPY . . RUN cargo install --root / --path . FROM debian:buster-slim EXPOSE 8080 WORKDIR /app +RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=builder /bin/fuzzysearch /bin/fuzzysearch CMD ["/bin/fuzzysearch"] diff --git a/src/filters.rs b/src/filters.rs index dabbd04..4cfb4a0 100644 --- a/src/filters.rs +++ b/src/filters.rs @@ -10,9 +10,10 @@ pub fn search( ) -> impl Filter + Clone { search_image(db.clone(), tree.clone()) .or(search_hashes(db.clone(), tree.clone())) - .or(stream_search_image(db.clone(), tree)) + .or(stream_search_image(db.clone(), tree.clone())) .or(search_file(db.clone())) - .or(check_handle(db)) + .or(check_handle(db.clone())) + .or(search_image_by_url(db, tree)) } pub fn search_file(db: Pool) -> impl Filter + Clone { @@ -54,6 +55,19 @@ pub fn search_image( }) } +pub fn search_image_by_url( + db: Pool, + tree: Tree, +) -> impl Filter + Clone { + warp::path("url") + .and(warp::get()) + .and(warp::query::()) + .and(with_pool(db)) + .and(with_tree(tree)) + .and(with_api_key()) + .and_then(handlers::search_image_by_url) +} + pub fn search_hashes( db: Pool, tree: Tree, diff --git a/src/handlers.rs b/src/handlers.rs index c28b3b2..d45e73a 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -1,27 +1,56 @@ use crate::models::{image_query, image_query_sync}; use crate::types::*; -use crate::{rate_limit, Pool, Tree}; +use crate::{early_return, rate_limit, Pool, Tree}; +use std::convert::TryInto; use tracing::{span, warn}; use tracing_futures::Instrument; -use warp::{reject, Rejection, Reply}; - -fn map_bb8_err(err: bb8::RunError) -> Rejection { - reject::custom(Error::from(err)) -} - -fn map_postgres_err(err: tokio_postgres::Error) -> Rejection { - reject::custom(Error::from(err)) -} +use warp::{Rejection, Reply}; #[derive(Debug)] enum Error { BB8(bb8::RunError), Postgres(tokio_postgres::Error), + Reqwest(reqwest::Error), InvalidData, + InvalidImage, ApiKey, RateLimit, } +impl warp::Reply for Error { + fn into_response(self) -> warp::reply::Response { + let msg = match self { + Error::BB8(_) | Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { + code: 500, + message: "Internal server error".to_string(), + }, + Error::InvalidData => ErrorMessage { + code: 400, + message: "Invalid data provided".to_string(), + }, + Error::InvalidImage => ErrorMessage { + code: 400, + message: "Invalid image provided".to_string(), + }, + Error::ApiKey => ErrorMessage { + code: 401, + message: "Invalid API key".to_string(), + }, + Error::RateLimit => ErrorMessage { + code: 429, + message: "Too many requests".to_string(), + }, + }; + + let body = hyper::body::Body::from(serde_json::to_string(&msg).unwrap()); + + warp::http::Response::builder() + .status(msg.code) + .body(body) + .unwrap() + } +} + impl From> for Error { fn from(err: bb8::RunError) -> Self { Error::BB8(err) @@ -34,12 +63,16 @@ impl From for Error { } } -impl warp::reject::Reject for Error {} +impl From for Error { + fn from(err: reqwest::Error) -> Self { + Error::Reqwest(err) + } +} #[tracing::instrument(skip(form))] async fn hash_input(form: warp::multipart::FormData) -> (i64, img_hash::ImageHash<[u8; 8]>) { use bytes::BufMut; - use futures_util::StreamExt; + use futures::StreamExt; let parts: Vec<_> = form.collect().await; let mut parts = parts @@ -82,11 +115,11 @@ pub async fn search_image( pool: Pool, tree: Tree, api_key: String, -) -> Result { - let db = pool.get().await.map_err(map_bb8_err)?; +) -> Result, Rejection> { + let db = early_return!(pool.get().await); - rate_limit!(&api_key, &db, image_limit, "image"); - rate_limit!(&api_key, &db, hash_limit, "hash"); + let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); + let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); let (num, hash) = hash_input(form).await; @@ -139,7 +172,20 @@ pub async fn search_image( matches: items, }; - Ok(warp::reply::json(&similarity)) + let resp = warp::http::Response::builder() + .header("x-image-hash", num.to_string()) + .header("x-rate-limit-total-image", image_remaining.1.to_string()) + .header( + "x-rate-limit-remaining-image", + image_remaining.0.to_string(), + ) + .header("x-rate-limit-total-hash", hash_remaining.1.to_string()) + .header("x-rate-limit-remaining-hash", hash_remaining.0.to_string()) + .header("content-type", "application/json") + .body(serde_json::to_string(&similarity).unwrap()) + .unwrap(); + + Ok(Box::new(resp)) } pub async fn stream_image( @@ -147,34 +193,38 @@ pub async fn stream_image( pool: Pool, tree: Tree, api_key: String, -) -> Result { - use futures_util::StreamExt; - - let db = pool.get().await.map_err(map_bb8_err)?; +) -> Result, Rejection> { + let db = early_return!(pool.get().await); rate_limit!(&api_key, &db, image_limit, "image", 2); rate_limit!(&api_key, &db, hash_limit, "hash"); let (num, hash) = hash_input(form).await; - let event_stream = image_query_sync( + let mut query = image_query_sync( pool.clone(), tree, vec![num], 10, Some(hash.as_bytes().to_vec()), - ) - .map(sse_matches); + ); - Ok(warp::sse::reply(event_stream)) + let event_stream = async_stream::stream! { + while let Some(result) = query.recv().await { + yield sse_matches(result); + } + }; + + Ok(Box::new(warp::sse::reply(event_stream))) } +#[allow(clippy::unnecessary_wraps)] fn sse_matches( matches: Result, tokio_postgres::Error>, -) -> Result { +) -> Result { let items = matches.unwrap(); - Ok(warp::sse::json(items)) + Ok(warp::sse::Event::default().json_data(items).unwrap()) } pub async fn search_hashes( @@ -182,9 +232,9 @@ pub async fn search_hashes( db: Pool, tree: Tree, api_key: String, -) -> Result { +) -> Result, Rejection> { let pool = db.clone(); - let db = db.get().await.map_err(map_bb8_err)?; + let db = early_return!(db.get().await); let hashes: Vec = opts .hashes @@ -194,10 +244,10 @@ pub async fn search_hashes( .collect(); if hashes.is_empty() { - return Err(warp::reject::custom(Error::InvalidData)); + return Ok(Box::new(Error::InvalidData)); } - rate_limit!(&api_key, &db, image_limit, "image", hashes.len() as i16); + let image_remaining = rate_limit!(&api_key, &db, image_limit, "image", hashes.len() as i16); let mut results = image_query_sync( pool, @@ -209,20 +259,30 @@ pub async fn search_hashes( let mut matches = Vec::new(); while let Some(r) = results.recv().await { - matches.extend(r.map_err(|e| warp::reject::custom(Error::Postgres(e)))?); + matches.extend(early_return!(r)); } - Ok(warp::reply::json(&matches)) + let resp = warp::http::Response::builder() + .header("x-rate-limit-total-image", image_remaining.1.to_string()) + .header( + "x-rate-limit-remaining-image", + image_remaining.0.to_string(), + ) + .header("content-type", "application/json") + .body(serde_json::to_string(&matches).unwrap()) + .unwrap(); + + Ok(Box::new(resp)) } pub async fn search_file( opts: FileSearchOpts, db: Pool, api_key: String, -) -> Result { - let db = db.get().await.map_err(map_bb8_err)?; +) -> Result, Rejection> { + let db = early_return!(db.get().await); - rate_limit!(&api_key, &db, name_limit, "file"); + let file_remaining = rate_limit!(&api_key, &db, name_limit, "file"); let (filter, val): (&'static str, &(dyn tokio_postgres::types::ToSql + Sync)) = if let Some(ref id) = opts.id { @@ -232,7 +292,7 @@ pub async fn search_file( } else if let Some(ref url) = opts.url { ("lower(url) = lower($1)", url) } else { - return Err(warp::reject::custom(Error::InvalidData)); + return Ok(Box::new(Error::InvalidData)); }; let query = format!( @@ -255,53 +315,143 @@ pub async fn search_file( filter ); - let matches: Vec<_> = db - .query::(&*query, &[val]) - .instrument(span!(tracing::Level::TRACE, "waiting for db")) - .await - .map_err(map_postgres_err)? - .into_iter() - .map(|row| File { - id: row.get("hash_id"), - site_id: row.get::<&str, i32>("id") as i64, - site_id_str: row.get::<&str, i32>("id").to_string(), - url: row.get("url"), - filename: row.get("filename"), - artists: row - .get::<&str, Option>("name") - .map(|artist| vec![artist]), - distance: None, - hash: None, - site_info: Some(SiteInfo::FurAffinity(FurAffinityFile { - file_id: row.get("file_id"), - })), - searched_hash: None, - }) - .collect(); + let matches: Vec<_> = early_return!( + db.query::(&*query, &[val]) + .instrument(span!(tracing::Level::TRACE, "waiting for db")) + .await + ) + .into_iter() + .map(|row| File { + id: row.get("hash_id"), + site_id: row.get::<&str, i32>("id") as i64, + site_id_str: row.get::<&str, i32>("id").to_string(), + url: row.get("url"), + filename: row.get("filename"), + artists: row + .get::<&str, Option>("name") + .map(|artist| vec![artist]), + distance: None, + hash: None, + site_info: Some(SiteInfo::FurAffinity(FurAffinityFile { + file_id: row.get("file_id"), + })), + searched_hash: None, + }) + .collect(); - Ok(warp::reply::json(&matches)) + let resp = warp::http::Response::builder() + .header("x-rate-limit-total-file", file_remaining.1.to_string()) + .header("x-rate-limit-remaining-file", file_remaining.0.to_string()) + .header("content-type", "application/json") + .body(serde_json::to_string(&matches).unwrap()) + .unwrap(); + + Ok(Box::new(resp)) } -pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result { - let db = db.get().await.map_err(map_bb8_err)?; +pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, Rejection> { + let db = early_return!(db.get().await); let exists = if let Some(handle) = opts.twitter { - !db.query( - "SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1)", - &[&handle], + !early_return!( + db.query( + "SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1)", + &[&handle], + ) + .await ) - .await - .map_err(map_postgres_err)? .is_empty() } else { false }; - Ok(warp::reply::json(&exists)) + Ok(Box::new(warp::reply::json(&exists))) +} + +pub async fn search_image_by_url( + opts: URLSearchOpts, + pool: Pool, + tree: Tree, + api_key: String, +) -> Result, Rejection> { + use bytes::BufMut; + + let url = opts.url; + + let db = early_return!(pool.get().await); + + let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); + let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); + + let mut resp = match reqwest::get(&url).await { + Ok(resp) => resp, + Err(_err) => return Ok(Box::new(Error::InvalidImage)), + }; + + let content_length = resp + .headers() + .get("content-length") + .and_then(|len| { + String::from_utf8_lossy(len.as_bytes()) + .parse::() + .ok() + }) + .unwrap_or(0); + + if content_length > 10_000_000 { + return Ok(Box::new(Error::InvalidImage)); + } + + let mut buf = bytes::BytesMut::with_capacity(content_length); + + while let Some(chunk) = early_return!(resp.chunk().await) { + if buf.len() + chunk.len() > 10_000_000 { + return Ok(Box::new(Error::InvalidImage)); + } + + buf.put(chunk); + } + + let hash = tokio::task::spawn_blocking(move || { + let hasher = crate::get_hasher(); + let image = image::load_from_memory(&buf).unwrap(); + hasher.hash_image(&image) + }) + .instrument(span!(tracing::Level::TRACE, "hashing image")) + .await + .unwrap(); + + let hash: [u8; 8] = hash.as_bytes().try_into().unwrap(); + let num = i64::from_be_bytes(hash); + + let results = image_query( + pool.clone(), + tree.clone(), + vec![num], + 3, + Some(hash.to_vec()), + ) + .await + .unwrap(); + + let resp = warp::http::Response::builder() + .header("x-image-hash", num.to_string()) + .header("x-rate-limit-total-image", image_remaining.1.to_string()) + .header( + "x-rate-limit-remaining-image", + image_remaining.0.to_string(), + ) + .header("x-rate-limit-total-hash", hash_remaining.1.to_string()) + .header("x-rate-limit-remaining-hash", hash_remaining.0.to_string()) + .header("content-type", "application/json") + .body(serde_json::to_string(&results).unwrap()) + .unwrap(); + + Ok(Box::new(resp)) } #[tracing::instrument] -pub async fn handle_rejection(err: Rejection) -> Result { +pub async fn handle_rejection(err: Rejection) -> Result, std::convert::Infallible> { warn!("had rejection"); let (code, message) = if err.is_not_found() { @@ -309,29 +459,10 @@ pub async fn handle_rejection(err: Rejection) -> Result() { - match err { - Error::BB8(_inner) => ( - warp::http::StatusCode::INTERNAL_SERVER_ERROR, - "A database error occured", - ), - Error::Postgres(_inner) => ( - warp::http::StatusCode::INTERNAL_SERVER_ERROR, - "A database error occured", - ), - Error::InvalidData => ( - warp::http::StatusCode::BAD_REQUEST, - "Unable to operate on provided data", - ), - Error::ApiKey => ( - warp::http::StatusCode::UNAUTHORIZED, - "Invalid API key provided", - ), - Error::RateLimit => ( - warp::http::StatusCode::TOO_MANY_REQUESTS, - "Your API token is rate limited", - ), - } + } else if err.find::().is_some() { + return Ok(Box::new(Error::InvalidData) as Box); + } else if err.find::().is_some() { + return Ok(Box::new(Error::InvalidData) as Box); } else { ( warp::http::StatusCode::INTERNAL_SERVER_ERROR, @@ -344,5 +475,5 @@ pub async fn handle_rejection(err: Rejection) -> Result $1", &[&id]) .await diff --git a/src/types.rs b/src/types.rs index 6106a6c..a4b76c8 100644 --- a/src/types.rs +++ b/src/types.rs @@ -20,7 +20,7 @@ pub enum RateLimit { /// This key is limited, we should deny the request. Limited, /// This key is available, contains the number of requests made. - Available(i16), + Available((i16, i16)), } /// A general type for every file. @@ -112,3 +112,8 @@ pub struct HashSearchOpts { pub struct HandleOpts { pub twitter: Option, } + +#[derive(Debug, Deserialize)] +pub struct URLSearchOpts { + pub url: String, +} diff --git a/src/utils.rs b/src/utils.rs index becb04f..404483d 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -7,18 +7,38 @@ macro_rules! rate_limit { rate_limit!($api_key, $db, $limit, $group, 1) }; - ($api_key:expr, $db:expr, $limit:tt, $group:expr, $incr_by:expr) => { - let api_key = crate::models::lookup_api_key($api_key, $db) - .await - .ok_or_else(|| warp::reject::custom(Error::ApiKey))?; + ($api_key:expr, $db:expr, $limit:tt, $group:expr, $incr_by:expr) => {{ + let api_key = match crate::models::lookup_api_key($api_key, $db).await { + Some(api_key) => api_key, + None => return Ok(Box::new(Error::ApiKey)), + }; - let rate_limit = - crate::utils::update_rate_limit($db, api_key.id, api_key.$limit, $group, $incr_by) - .await - .map_err(crate::handlers::map_postgres_err)?; + let rate_limit = match crate::utils::update_rate_limit( + $db, + api_key.id, + api_key.$limit, + $group, + $incr_by, + ) + .await + { + Ok(rate_limit) => rate_limit, + Err(err) => return Ok(Box::new(Error::Postgres(err))), + }; - if rate_limit == crate::types::RateLimit::Limited { - return Err(warp::reject::custom(Error::RateLimit)); + match rate_limit { + crate::types::RateLimit::Limited => return Ok(Box::new(Error::RateLimit)), + crate::types::RateLimit::Available(count) => count, + } + }}; +} + +#[macro_export] +macro_rules! early_return { + ($val:expr) => { + match $val { + Ok(val) => val, + Err(err) => return Ok(Box::new(Error::from(err))), } }; } @@ -59,14 +79,17 @@ pub async fn update_rate_limit( if count > key_group_limit { Ok(RateLimit::Limited) } else { - Ok(RateLimit::Available(count)) + Ok(RateLimit::Available(( + key_group_limit - count, + key_group_limit, + ))) } } -pub fn extract_rows<'a>( +pub fn extract_rows( rows: Vec, - hash: Option<&'a [u8]>, -) -> impl IntoIterator + 'a { + hash: Option<&[u8]>, +) -> impl IntoIterator + '_ { rows.into_iter().map(move |row| { let dbhash: i64 = row.get("hash"); let dbbytes = dbhash.to_be_bytes(); From 06a1c7b46632e3bf7144c21e5241b53b3d75d8cf Mon Sep 17 00:00:00 2001 From: Syfaro Date: Wed, 17 Feb 2021 12:28:58 -0500 Subject: [PATCH 2/6] Minor formatting changes. --- src/filters.rs | 2 +- src/handlers.rs | 8 ++-- src/main.rs | 112 ++++++++++++++++++++++++------------------------ src/models.rs | 4 +- src/types.rs | 2 +- src/utils.rs | 4 +- 6 files changed, 65 insertions(+), 67 deletions(-) diff --git a/src/filters.rs b/src/filters.rs index 4cfb4a0..4d734ef 100644 --- a/src/filters.rs +++ b/src/filters.rs @@ -61,7 +61,7 @@ pub fn search_image_by_url( ) -> impl Filter + Clone { warp::path("url") .and(warp::get()) - .and(warp::query::()) + .and(warp::query::()) .and(with_pool(db)) .and(with_tree(tree)) .and(with_api_key()) diff --git a/src/handlers.rs b/src/handlers.rs index d45e73a..f326a51 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -8,7 +8,7 @@ use warp::{Rejection, Reply}; #[derive(Debug)] enum Error { - BB8(bb8::RunError), + Bb8(bb8::RunError), Postgres(tokio_postgres::Error), Reqwest(reqwest::Error), InvalidData, @@ -20,7 +20,7 @@ enum Error { impl warp::Reply for Error { fn into_response(self) -> warp::reply::Response { let msg = match self { - Error::BB8(_) | Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { + Error::Bb8(_) | Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { code: 500, message: "Internal server error".to_string(), }, @@ -53,7 +53,7 @@ impl warp::Reply for Error { impl From> for Error { fn from(err: bb8::RunError) -> Self { - Error::BB8(err) + Error::Bb8(err) } } @@ -369,7 +369,7 @@ pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, } pub async fn search_image_by_url( - opts: URLSearchOpts, + opts: UrlSearchOpts, pool: Pool, tree: Tree, api_key: String, diff --git a/src/main.rs b/src/main.rs index 6ccbf30..a774b13 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,6 +3,7 @@ use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; +use warp::Filter; mod filters; mod handlers; @@ -10,60 +11,8 @@ mod models; mod types; mod utils; -use warp::Filter; - -fn configure_tracing() { - use opentelemetry::{ - api::{KeyValue, Provider}, - sdk::{Config, Sampler}, - }; - use tracing_subscriber::{layer::SubscriberExt, prelude::*}; - - let env = if cfg!(debug_assertions) { - "debug" - } else { - "release" - }; - - let fmt_layer = tracing_subscriber::fmt::layer(); - let filter_layer = tracing_subscriber::EnvFilter::try_from_default_env() - .or_else(|_| tracing_subscriber::EnvFilter::try_new("info")) - .unwrap(); - tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) - .finish(); - let registry = tracing_subscriber::registry() - .with(filter_layer) - .with(fmt_layer); - - let exporter = opentelemetry_jaeger::Exporter::builder() - .with_agent_endpoint(std::env::var("JAEGER_COLLECTOR").unwrap().parse().unwrap()) - .with_process(opentelemetry_jaeger::Process { - service_name: "fuzzysearch".to_string(), - tags: vec![ - KeyValue::new("environment", env), - KeyValue::new("version", env!("CARGO_PKG_VERSION")), - ], - }) - .init() - .expect("unable to create jaeger exporter"); - - let provider = opentelemetry::sdk::Provider::builder() - .with_simple_exporter(exporter) - .with_config(Config { - default_sampler: Box::new(Sampler::Always), - ..Default::default() - }) - .build(); - - opentelemetry::global::set_provider(provider); - - let tracer = opentelemetry::global::trace_provider().get_tracer("fuzzysearch"); - let telem_layer = tracing_opentelemetry::layer().with_tracer(tracer); - let registry = registry.with(telem_layer); - - registry.init(); -} +type Tree = Arc>>; +type Pool = bb8::Pool>; #[derive(Debug)] pub struct Node { @@ -77,8 +26,6 @@ impl Node { } } -type Tree = Arc>>; - pub struct Hamming; impl bk_tree::Metric for Hamming { @@ -198,7 +145,58 @@ async fn main() { warp::serve(routes).run(([0, 0, 0, 0], 8080)).await; } -type Pool = bb8::Pool>; +fn configure_tracing() { + use opentelemetry::{ + api::{KeyValue, Provider}, + sdk::{Config, Sampler}, + }; + use tracing_subscriber::{layer::SubscriberExt, prelude::*}; + + let env = if cfg!(debug_assertions) { + "debug" + } else { + "release" + }; + + let fmt_layer = tracing_subscriber::fmt::layer(); + let filter_layer = tracing_subscriber::EnvFilter::try_from_default_env() + .or_else(|_| tracing_subscriber::EnvFilter::try_new("info")) + .unwrap(); + tracing_subscriber::fmt() + .with_env_filter(tracing_subscriber::EnvFilter::from_default_env()) + .finish(); + let registry = tracing_subscriber::registry() + .with(filter_layer) + .with(fmt_layer); + + let exporter = opentelemetry_jaeger::Exporter::builder() + .with_agent_endpoint(std::env::var("JAEGER_COLLECTOR").unwrap().parse().unwrap()) + .with_process(opentelemetry_jaeger::Process { + service_name: "fuzzysearch".to_string(), + tags: vec![ + KeyValue::new("environment", env), + KeyValue::new("version", env!("CARGO_PKG_VERSION")), + ], + }) + .init() + .expect("unable to create jaeger exporter"); + + let provider = opentelemetry::sdk::Provider::builder() + .with_simple_exporter(exporter) + .with_config(Config { + default_sampler: Box::new(Sampler::Always), + ..Default::default() + }) + .build(); + + opentelemetry::global::set_provider(provider); + + let tracer = opentelemetry::global::trace_provider().get_tracer("fuzzysearch"); + let telem_layer = tracing_opentelemetry::layer().with_tracer(tracer); + let registry = registry.with(telem_layer); + + registry.init(); +} fn get_hasher() -> img_hash::Hasher<[u8; 8]> { use img_hash::{HashAlg::Gradient, HasherConfig}; diff --git a/src/models.rs b/src/models.rs index 2bd4fb0..7fc7a58 100644 --- a/src/models.rs +++ b/src/models.rs @@ -3,11 +3,11 @@ use crate::utils::extract_rows; use crate::{Pool, Tree}; use tracing_futures::Instrument; -pub type DB<'a> = +pub type Db<'a> = &'a bb8::PooledConnection<'a, bb8_postgres::PostgresConnectionManager>; #[tracing::instrument(skip(db))] -pub async fn lookup_api_key(key: &str, db: DB<'_>) -> Option { +pub async fn lookup_api_key(key: &str, db: Db<'_>) -> Option { let rows = db .query( "SELECT diff --git a/src/types.rs b/src/types.rs index a4b76c8..55d6187 100644 --- a/src/types.rs +++ b/src/types.rs @@ -114,6 +114,6 @@ pub struct HandleOpts { } #[derive(Debug, Deserialize)] -pub struct URLSearchOpts { +pub struct UrlSearchOpts { pub url: String, } diff --git a/src/utils.rs b/src/utils.rs index 404483d..d0e940e 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,4 @@ -use crate::models::DB; +use crate::models::Db; use crate::types::*; #[macro_export] @@ -51,7 +51,7 @@ macro_rules! early_return { /// joined requests. #[tracing::instrument(skip(db))] pub async fn update_rate_limit( - db: DB<'_>, + db: Db<'_>, key_id: i32, key_group_limit: i16, group_name: &'static str, From 908cda8ce9a53ef0da6b94ba3a760ec6d58493b1 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Wed, 17 Feb 2021 16:30:05 -0500 Subject: [PATCH 3/6] Use NOTIFY/LISTEN instead of polling for updates (#3) * Use NOTIFY/LISTEN instead of polling for updates. * Allow different distances for multiple hashes. --- .gitignore | 1 + Cargo.lock | 678 +++++++++++++++++++++++++++++++----------------- Cargo.toml | 4 +- Dockerfile | 1 + sqlx-data.json | 194 ++++++++++++++ src/handlers.rs | 201 +++++++------- src/main.rs | 152 +++++------ src/models.rs | 102 +++++--- src/types.rs | 2 +- src/utils.rs | 85 ++---- 10 files changed, 891 insertions(+), 529 deletions(-) create mode 100644 sqlx-data.json diff --git a/.gitignore b/.gitignore index ea8c4bf..fedaa2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +.env diff --git a/Cargo.lock b/Cargo.lock index ee5443e..2bfdf25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler" version = "0.2.3" @@ -12,6 +14,32 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + +[[package]] +name = "ahash" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796540673305a66d127804eef19ad696f1f204b8c1025aaca4958c17eab32877" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -21,6 +49,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "async-stream" version = "0.3.0" @@ -43,14 +77,12 @@ dependencies = [ ] [[package]] -name = "async-trait" -version = "0.1.42" +name = "atoi" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" dependencies = [ - "proc-macro2", - "quote", - "syn", + "num-traits", ] [[package]] @@ -71,37 +103,24 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" -[[package]] -name = "bb8" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dae93eccab998c4b8703e3a6bbaa1714c38e445ebacb4bede25d0408521e293c" -dependencies = [ - "async-trait", - "futures-channel", - "futures-util", - "parking_lot", - "tokio", -] - -[[package]] -name = "bb8-postgres" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fdf56d52b2cca401d2380407e5c35d3d25d3560224ecf74d6e4ca13e51239b" -dependencies = [ - "async-trait", - "bb8", - "tokio", - "tokio-postgres", -] - [[package]] name = "bitflags" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitvec" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "bk-tree" version = "0.3.0" @@ -149,10 +168,16 @@ dependencies = [ ] [[package]] -name = "bumpalo" -version = "3.4.0" +name = "build_const" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" + +[[package]] +name = "bumpalo" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099e596ef14349721d9016f6b80dd3419ea1bf289ab9b44df8e4dfd3a005d5d9" [[package]] name = "byte-tools" @@ -243,6 +268,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + [[package]] name = "crc32fast" version = "1.2.1" @@ -287,6 +321,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.1" @@ -336,17 +380,26 @@ dependencies = [ "generic-array 0.14.4", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" -version = "0.8.26" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" dependencies = [ "cfg-if 1.0.0", ] @@ -357,12 +410,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fnv" version = "1.0.7" @@ -394,6 +441,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + [[package]] name = "futures" version = "0.3.12" @@ -494,8 +547,6 @@ name = "fuzzysearch" version = "0.1.0" dependencies = [ "async-stream", - "bb8", - "bb8-postgres", "bk-tree", "bytes", "chrono", @@ -509,8 +560,8 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sqlx", "tokio", - "tokio-postgres", "tracing", "tracing-futures", "tracing-opentelemetry", @@ -556,7 +607,7 @@ checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.1+wasi-snapshot-preview1", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -600,6 +651,18 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash 0.4.7", +] + +[[package]] +name = "hashlink" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +dependencies = [ + "hashbrown", +] [[package]] name = "headers" @@ -626,6 +689,15 @@ dependencies = [ "http", ] +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -635,6 +707,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "hmac" version = "0.10.1" @@ -668,9 +746,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" [[package]] name = "httpdate" @@ -680,9 +758,9 @@ checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "hyper" -version = "0.14.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12219dc884514cb4a6a03737f4413c0e01c23a1b059b0156004b23f1e19dccbe" +checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" dependencies = [ "bytes", "futures-channel", @@ -694,7 +772,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.4", + "pin-project 1.0.5", "socket2", "tokio", "tower-service", @@ -717,9 +795,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "de910d521f7cc3135c4de8db1cb910e0b5ed1dc6f57c381cd07e8e661ce10094" dependencies = [ "matches", "unicode-bidi", @@ -728,9 +806,9 @@ dependencies = [ [[package]] name = "image" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce04077ead78e39ae8610ad26216aed811996b043d47beed5090db674f9e9b5" +checksum = "293f07a1875fa7e9c5897b51aa68b2d8ed8271b87e1a44cb64b9c3d98aabbc0d" dependencies = [ "bytemuck", "byteorder", @@ -788,9 +866,9 @@ dependencies = [ [[package]] name = "integer-encoding" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6104619c35f8835695e517cfb80fb7142139ee4b53f4d0fa4c8dca6e98fbc66" +checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" [[package]] name = "ipnet" @@ -806,19 +884,18 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jpeg-decoder" -version = "0.1.20" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc797adac5f083b8ff0ca6f6294a999393d76e197c36488e2ef732c4715f6fa3" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" dependencies = [ - "byteorder", "rayon", ] [[package]] name = "js-sys" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +checksum = "5cfb73131c35423a367daf8cbd24100af0d077668c8c2943f0e7dd775fef0f65" dependencies = [ "wasm-bindgen", ] @@ -830,10 +907,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "libc" -version = "0.2.82" +name = "lexical-core" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" +checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if 1.0.0", + "ryu", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "lock_api" @@ -846,13 +936,19 @@ dependencies = [ [[package]] name = "log" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.0.1" @@ -869,10 +965,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] -name = "md5" -version = "0.7.0" +name = "md-5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug 0.3.0", +] [[package]] name = "memchr" @@ -983,6 +1084,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nom" +version = "6.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2" +dependencies = [ + "bitvec", + "funty", + "lexical-core", + "memchr", + "version_check", +] + [[package]] name = "ntapi" version = "0.3.6" @@ -1150,14 +1264,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "smallvec", "winapi", ] @@ -1168,24 +1282,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project" version = "0.4.27" @@ -1197,11 +1293,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" +checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" dependencies = [ - "pin-project-internal 1.0.4", + "pin-project-internal 1.0.5", ] [[package]] @@ -1217,9 +1313,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" +checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ "proc-macro2", "quote", @@ -1256,35 +1352,6 @@ dependencies = [ "miniz_oxide 0.3.7", ] -[[package]] -name = "postgres-protocol" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e34ad3dc5c56d036b9418185ee97e14b6766d55c8ccf9dc18302ad4e6371d9" -dependencies = [ - "base64 0.13.0", - "byteorder", - "bytes", - "fallible-iterator", - "hmac", - "md5", - "memchr", - "rand 0.8.2", - "sha2", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5493d9d4613b88b12433aa12890e74e74cd93fdc1e08b7c2aed4768aaae8414c" -dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol", -] - [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1328,9 +1395,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.20.0" +version = "2.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86473d5f16580f10b131a0bf0afb68f8e029d1835d33a00f37281b05694e5312" +checksum = "73f72884896d22e0da0e5b266cb9a780b791f6c3b2f5beab6368d6cd4f0dbb86" [[package]] name = "quick-error" @@ -1340,13 +1407,19 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + [[package]] name = "rand" version = "0.7.3" @@ -1362,13 +1435,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18519b42a40024d661e1714153e9ad0c3de27cd495760ceb09710920f1098b1e" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" dependencies = [ "libc", "rand_chacha 0.3.0", - "rand_core 0.6.1", + "rand_core 0.6.2", "rand_hc 0.3.0", ] @@ -1389,7 +1462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ "ppv-lite86", - "rand_core 0.6.1", + "rand_core 0.6.2", ] [[package]] @@ -1403,9 +1476,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ "getrandom 0.2.2", ] @@ -1425,7 +1498,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" dependencies = [ - "rand_core 0.6.1", + "rand_core 0.6.2", ] [[package]] @@ -1455,15 +1528,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - -[[package]] -name = "redox_syscall" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" dependencies = [ "bitflags", ] @@ -1474,7 +1541,10 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", + "thread_local", ] [[package]] @@ -1623,18 +1693,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.120" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166b2349061381baf54a58e4b13c89369feb0ef2eaa57198899e2312aac30aab" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.120" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca2a8cb5805ce9e3b95435e3765b7b553cecc762d938d409434338386cb5775" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" dependencies = [ "proc-macro2", "quote", @@ -1643,10 +1713,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486" dependencies = [ + "indexmap", "itoa", "ryu", "serde", @@ -1678,9 +1749,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -1691,9 +1762,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -1720,12 +1791,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" - [[package]] name = "slab" version = "0.4.2" @@ -1755,6 +1820,116 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "sqlformat" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74c70f0235b9925cbb106c52af1a28b5ea4885a8b851e328b8562e257a389c2d" +dependencies = [ + "lazy_static", + "maplit", + "nom", + "regex", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2739d54a2ae9fdd0f545cb4e4b5574efb95e2ec71b7f921678e246fb20dcaaf" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1cad9cae4ca8947eba1a90e8ec7d3c59e7a768e2f120dc9013b669c34a90711" +dependencies = [ + "ahash 0.6.3", + "atoi", + "base64 0.13.0", + "bitflags", + "byteorder", + "bytes", + "crc", + "crossbeam-channel", + "crossbeam-queue", + "crossbeam-utils", + "either", + "futures-channel", + "futures-core", + "futures-util", + "hashlink", + "hex", + "hmac", + "itoa", + "libc", + "log", + "md-5", + "memchr", + "once_cell", + "parking_lot", + "percent-encoding", + "rand 0.7.3", + "serde", + "serde_json", + "sha-1 0.9.4", + "sha2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "tokio-stream", + "url", + "whoami", +] + +[[package]] +name = "sqlx-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01caee2b3935b4efe152f3262afbe51546ce3b1fc27ad61014e1b3cf5f55366e" +dependencies = [ + "dotenv", + "either", + "futures", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-rt", + "syn", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce2e16b6774c671cc183e1d202386fdf9cde1e8468c1894a7f2a63eb671c4f4" +dependencies = [ + "native-tls", + "once_cell", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strength_reduce" version = "0.2.3" @@ -1779,15 +1954,21 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.2.0" @@ -1796,19 +1977,39 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.2", - "redox_syscall 0.2.4", + "rand 0.8.3", + "redox_syscall", "remove_dir_all", "winapi", ] [[package]] -name = "thread_local" -version = "1.1.0" +name = "thiserror" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447" +checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146" dependencies = [ - "lazy_static", + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", ] [[package]] @@ -1856,9 +2057,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" dependencies = [ "tinyvec_macros", ] @@ -1871,9 +2072,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.0.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca04cec6ff2474c638057b65798f60ac183e5e79d3448bb7163d36a39cff6ec" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" dependencies = [ "autocfg", "bytes", @@ -1891,9 +2092,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42517d2975ca3114b22a16192634e8241dc5cc1f130be194645970cc1c371494" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2", "quote", @@ -1910,34 +2111,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-postgres" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc9f82c2bfb06a33dd0dfb44b07ca98fe72df19e681d80c78d05a1bac2138e2" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures", - "log", - "parking_lot", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "socket2", - "tokio", - "tokio-util", -] - [[package]] name = "tokio-stream" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76066865172052eb8796c686f0b441a93df8b08d40a950b062ffb9a426f00edd" +checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" dependencies = [ "futures-core", "pin-project-lite", @@ -1952,38 +2130,36 @@ checksum = "e1a5f475f1b9d077ea1017ecbc60890fda8e54942d680ca0b1d2b47cfa2d861b" dependencies = [ "futures-util", "log", - "pin-project 1.0.4", + "pin-project 1.0.5", "tokio", "tungstenite", ] [[package]] name = "tokio-util" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb971a26599ffd28066d387f109746df178eff14d5ea1e235015c5601967a4b" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" dependencies = [ - "async-stream", "bytes", "futures-core", "futures-sink", "log", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "tower-service" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "f7d40a22fd029e33300d8d89a5cc8ffce18bb7c587662f54629e94c9de5487f3" dependencies = [ "cfg-if 1.0.0", "log", @@ -1994,9 +2170,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" +checksum = "43f080ea7e4107844ef4766459426fa2d5c1ada2e47edba05dc7fa99d9629f47" dependencies = [ "proc-macro2", "quote", @@ -2014,11 +2190,11 @@ dependencies = [ [[package]] name = "tracing-futures" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 0.4.27", + "pin-project 1.0.5", "tracing", ] @@ -2114,8 +2290,8 @@ dependencies = [ "httparse", "input_buffer", "log", - "rand 0.8.2", - "sha-1 0.9.2", + "rand 0.8.3", + "sha-1 0.9.4", "url", "utf-8", ] @@ -2155,19 +2331,31 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + [[package]] name = "unicode-xid" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "url" version = "2.2.0" @@ -2224,7 +2412,7 @@ dependencies = [ "mime_guess", "multipart", "percent-encoding", - "pin-project 1.0.4", + "pin-project 1.0.5", "scoped-tls", "serde", "serde_json", @@ -2246,15 +2434,15 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.1+wasi-snapshot-preview1" +version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93c6c3420963c5c64bca373b25e77acb562081b9bb4dd5bb864187742186cea9" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +checksum = "55c0f7123de74f0dab9b7d00fd614e7b19349cd1e2f5252bbe9b1754b59433be" dependencies = [ "cfg-if 1.0.0", "serde", @@ -2264,9 +2452,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +checksum = "7bc45447f0d4573f3d65720f636bbcc3dd6ce920ed704670118650bcd47764c7" dependencies = [ "bumpalo", "lazy_static", @@ -2279,9 +2467,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "3de431a2910c86679c34283a33f66f4e4abd7e0aec27b6669060148872aadf94" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -2291,9 +2479,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +checksum = "3b8853882eef39593ad4174dd26fc9865a64e84026d223f63bb2c42affcbba2c" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2301,9 +2489,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +checksum = "4133b5e7f2a531fa413b3a1695e925038a05a71cf67e87dafa295cb645a01385" dependencies = [ "proc-macro2", "quote", @@ -2314,15 +2502,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64" [[package]] name = "web-sys" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +checksum = "c40dc691fc48003eba817c38da7113c15698142da971298003cac3ef175680b3" dependencies = [ "js-sys", "wasm-bindgen", @@ -2330,9 +2518,19 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277" +checksum = "4a32b378380f4e9869b22f0b5177c68a5519f03b3454fde0b291455ddbae266c" + +[[package]] +name = "whoami" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a921c0ad578a51c0b6c0bbb9b95f0ed11e90d61da506139e48a946edd11ee1e" +dependencies = [ + "wasm-bindgen", + "web-sys", +] [[package]] name = "winapi" @@ -2364,3 +2562,9 @@ checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi", ] + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" diff --git a/Cargo.toml b/Cargo.toml index 02404a6..4181e74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,9 +28,7 @@ warp = "0.3" reqwest = "0.11" hyper = "0.14" -tokio-postgres = "0.7" -bb8 = "0.7" -bb8-postgres = "0.7" +sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "macros", "json", "offline"] } img_hash = "3" image = "0.23" diff --git a/Dockerfile b/Dockerfile index 46e231d..a9cb0e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM rust:1-slim AS builder WORKDIR /src +ENV SQLX_OFFLINE=true RUN apt-get update -y && apt-get install -y libssl-dev pkg-config COPY . . RUN cargo install --root / --path . diff --git a/sqlx-data.json b/sqlx-data.json new file mode 100644 index 0000000..db41156 --- /dev/null +++ b/sqlx-data.json @@ -0,0 +1,194 @@ +{ + "db": "PostgreSQL", + "1984ce60f052d6a29638f8e05b35671b8edfbf273783d4b843ebd35cbb8a391f": { + "query": "INSERT INTO\n rate_limit (api_key_id, time_window, group_name, count)\n VALUES\n ($1, $2, $3, $4)\n ON CONFLICT ON CONSTRAINT unique_window\n DO UPDATE set count = rate_limit.count + $4\n RETURNING rate_limit.count", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int2" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int8", + "Text", + "Int2" + ] + }, + "nullable": [ + false + ] + } + }, + "659ee9ddc1c5ccd42ba9dc1617440544c30ece449ba3ba7f9d39f447b8af3cfe": { + "query": "SELECT\n api_key.id,\n api_key.name_limit,\n api_key.image_limit,\n api_key.hash_limit,\n api_key.name,\n account.email owner_email\n FROM\n api_key\n JOIN account\n ON account.id = api_key.user_id\n WHERE\n api_key.key = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "name_limit", + "type_info": "Int2" + }, + { + "ordinal": 2, + "name": "image_limit", + "type_info": "Int2" + }, + { + "ordinal": 3, + "name": "hash_limit", + "type_info": "Int2" + }, + { + "ordinal": 4, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "owner_email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false + ] + } + }, + "6b8d304fc40fa539ae671e6e24e7978ad271cb7a1cafb20fc4b4096a958d790f": { + "query": "SELECT exists(SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + } + }, + "f4608ccaf739d36649cdbc5297177a989cc7763006d28c97e219bb708930972a": { + "query": "SELECT\n hashes.id,\n hashes.hash,\n hashes.furaffinity_id,\n hashes.e621_id,\n hashes.twitter_id,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.url)\n WHEN e621_id IS NOT NULL THEN (e.data->'file'->>'url')\n WHEN twitter_id IS NOT NULL THEN (tm.url)\n END url,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.filename)\n WHEN e621_id IS NOT NULL THEN ((e.data->'file'->>'md5') || '.' || (e.data->'file'->>'ext'))\n WHEN twitter_id IS NOT NULL THEN (SELECT split_part(split_part(tm.url, '/', 5), ':', 1))\n END filename,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (ARRAY(SELECT f.name))\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'tags'->'artist'))\n WHEN twitter_id IS NOT NULL THEN ARRAY(SELECT tw.data->'user'->>'screen_name')\n END artists,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.file_id)\n END file_id,\n CASE\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'sources'))\n END sources\n FROM\n hashes\n LEFT JOIN LATERAL (\n SELECT *\n FROM submission\n JOIN artist ON submission.artist_id = artist.id\n WHERE submission.id = hashes.furaffinity_id\n ) f ON hashes.furaffinity_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM e621\n WHERE e621.id = hashes.e621_id\n ) e ON hashes.e621_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet\n WHERE tweet.id = hashes.twitter_id\n ) tw ON hashes.twitter_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet_media\n WHERE\n tweet_media.tweet_id = hashes.twitter_id AND\n tweet_media.hash <@ (hashes.hash, 0)\n LIMIT 1\n ) tm ON hashes.twitter_id IS NOT NULL\n WHERE hashes.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "furaffinity_id", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "e621_id", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "twitter_id", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "url", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "filename", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "artists", + "type_info": "TextArray" + }, + { + "ordinal": 8, + "name": "file_id", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "sources", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + null, + null, + null, + null, + null + ] + } + }, + "fe60be66b2d8a8f02b3bfe06d1f0e57e4bb07e80cba1b379a5f17f6cbd8b075c": { + "query": "SELECT id, hash FROM hashes", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + } + } +} \ No newline at end of file diff --git a/src/handlers.rs b/src/handlers.rs index f326a51..fdbe4e9 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -8,8 +8,7 @@ use warp::{Rejection, Reply}; #[derive(Debug)] enum Error { - Bb8(bb8::RunError), - Postgres(tokio_postgres::Error), + Postgres(sqlx::Error), Reqwest(reqwest::Error), InvalidData, InvalidImage, @@ -20,7 +19,7 @@ enum Error { impl warp::Reply for Error { fn into_response(self) -> warp::reply::Response { let msg = match self { - Error::Bb8(_) | Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { + Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { code: 500, message: "Internal server error".to_string(), }, @@ -51,14 +50,8 @@ impl warp::Reply for Error { } } -impl From> for Error { - fn from(err: bb8::RunError) -> Self { - Error::Bb8(err) - } -} - -impl From for Error { - fn from(err: tokio_postgres::Error) -> Self { +impl From for Error { + fn from(err: sqlx::Error) -> Self { Error::Postgres(err) } } @@ -112,12 +105,10 @@ async fn hash_input(form: warp::multipart::FormData) -> (i64, img_hash::ImageHas pub async fn search_image( form: warp::multipart::FormData, opts: ImageSearchOpts, - pool: Pool, + db: Pool, tree: Tree, api_key: String, ) -> Result, Rejection> { - let db = early_return!(pool.get().await); - let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); @@ -126,7 +117,7 @@ pub async fn search_image( let mut items = { if opts.search_type == Some(ImageSearchType::Force) { image_query( - pool.clone(), + db.clone(), tree.clone(), vec![num], 10, @@ -136,7 +127,7 @@ pub async fn search_image( .unwrap() } else { let results = image_query( - pool.clone(), + db.clone(), tree.clone(), vec![num], 0, @@ -146,7 +137,7 @@ pub async fn search_image( .unwrap(); if results.is_empty() && opts.search_type != Some(ImageSearchType::Exact) { image_query( - pool.clone(), + db.clone(), tree.clone(), vec![num], 10, @@ -194,10 +185,8 @@ pub async fn stream_image( tree: Tree, api_key: String, ) -> Result, Rejection> { - let db = early_return!(pool.get().await); - - rate_limit!(&api_key, &db, image_limit, "image", 2); - rate_limit!(&api_key, &db, hash_limit, "hash"); + rate_limit!(&api_key, &pool, image_limit, "image", 2); + rate_limit!(&api_key, &pool, hash_limit, "hash"); let (num, hash) = hash_input(form).await; @@ -220,7 +209,7 @@ pub async fn stream_image( #[allow(clippy::unnecessary_wraps)] fn sse_matches( - matches: Result, tokio_postgres::Error>, + matches: Result, sqlx::Error>, ) -> Result { let items = matches.unwrap(); @@ -234,7 +223,6 @@ pub async fn search_hashes( api_key: String, ) -> Result, Rejection> { let pool = db.clone(); - let db = early_return!(db.get().await); let hashes: Vec = opts .hashes @@ -280,64 +268,95 @@ pub async fn search_file( db: Pool, api_key: String, ) -> Result, Rejection> { - let db = early_return!(db.get().await); + use sqlx::Row; let file_remaining = rate_limit!(&api_key, &db, name_limit, "file"); - let (filter, val): (&'static str, &(dyn tokio_postgres::types::ToSql + Sync)) = - if let Some(ref id) = opts.id { - ("file_id = $1", id) - } else if let Some(ref name) = opts.name { - ("lower(filename) = lower($1)", name) - } else if let Some(ref url) = opts.url { - ("lower(url) = lower($1)", url) - } else { - return Ok(Box::new(Error::InvalidData)); - }; + let query = if let Some(ref id) = opts.id { + sqlx::query( + "SELECT + submission.id, + submission.url, + submission.filename, + submission.file_id, + artist.name, + hashes.id hash_id + FROM + submission + JOIN artist + ON artist.id = submission.artist_id + JOIN hashes + ON hashes.furaffinity_id = submission.id + WHERE + file_id = $1 + LIMIT 10", + ) + .bind(id) + } else if let Some(ref name) = opts.name { + sqlx::query( + "SELECT + submission.id, + submission.url, + submission.filename, + submission.file_id, + artist.name, + hashes.id hash_id + FROM + submission + JOIN artist + ON artist.id = submission.artist_id + JOIN hashes + ON hashes.furaffinity_id = submission.id + WHERE + lower(filename) = lower($1) + LIMIT 10", + ) + .bind(name) + } else if let Some(ref url) = opts.url { + sqlx::query( + "SELECT + submission.id, + submission.url, + submission.filename, + submission.file_id, + artist.name, + hashes.id hash_id + FROM + submission + JOIN artist + ON artist.id = submission.artist_id + JOIN hashes + ON hashes.furaffinity_id = submission.id + WHERE + lower(url) = lower($1) + LIMIT 10", + ) + .bind(url) + } else { + return Ok(Box::new(Error::InvalidData)); + }; - let query = format!( - "SELECT - submission.id, - submission.url, - submission.filename, - submission.file_id, - artist.name, - hashes.id hash_id - FROM - submission - JOIN artist - ON artist.id = submission.artist_id - JOIN hashes - ON hashes.furaffinity_id = submission.id - WHERE - {} - LIMIT 10", - filter - ); + let matches: Result, _> = query + .map(|row| File { + id: row.get("hash_id"), + site_id: row.get::("id") as i64, + site_id_str: row.get::("id").to_string(), + url: row.get("url"), + filename: row.get("filename"), + artists: row + .get::, _>("name") + .map(|artist| vec![artist]), + distance: None, + hash: None, + site_info: Some(SiteInfo::FurAffinity(FurAffinityFile { + file_id: row.get("file_id"), + })), + searched_hash: None, + }) + .fetch_all(&db) + .await; - let matches: Vec<_> = early_return!( - db.query::(&*query, &[val]) - .instrument(span!(tracing::Level::TRACE, "waiting for db")) - .await - ) - .into_iter() - .map(|row| File { - id: row.get("hash_id"), - site_id: row.get::<&str, i32>("id") as i64, - site_id_str: row.get::<&str, i32>("id").to_string(), - url: row.get("url"), - filename: row.get("filename"), - artists: row - .get::<&str, Option>("name") - .map(|artist| vec![artist]), - distance: None, - hash: None, - site_info: Some(SiteInfo::FurAffinity(FurAffinityFile { - file_id: row.get("file_id"), - })), - searched_hash: None, - }) - .collect(); + let matches = early_return!(matches); let resp = warp::http::Response::builder() .header("x-rate-limit-total-file", file_remaining.1.to_string()) @@ -350,17 +369,13 @@ pub async fn search_file( } pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, Rejection> { - let db = early_return!(db.get().await); - let exists = if let Some(handle) = opts.twitter { - !early_return!( - db.query( - "SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1)", - &[&handle], - ) + let result = sqlx::query_scalar!("SELECT exists(SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1))", handle) + .fetch_optional(&db) .await - ) - .is_empty() + .map(|row| row.flatten().unwrap_or(false)); + + early_return!(result) } else { false }; @@ -370,7 +385,7 @@ pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, pub async fn search_image_by_url( opts: UrlSearchOpts, - pool: Pool, + db: Pool, tree: Tree, api_key: String, ) -> Result, Rejection> { @@ -378,8 +393,6 @@ pub async fn search_image_by_url( let url = opts.url; - let db = early_return!(pool.get().await); - let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); @@ -424,15 +437,9 @@ pub async fn search_image_by_url( let hash: [u8; 8] = hash.as_bytes().try_into().unwrap(); let num = i64::from_be_bytes(hash); - let results = image_query( - pool.clone(), - tree.clone(), - vec![num], - 3, - Some(hash.to_vec()), - ) - .await - .unwrap(); + let results = image_query(db.clone(), tree.clone(), vec![num], 3, Some(hash.to_vec())) + .await + .unwrap(); let resp = warp::http::Response::builder() .header("x-image-hash", num.to_string()) diff --git a/src/main.rs b/src/main.rs index a774b13..f005325 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ #![recursion_limit = "256"] -use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; use warp::Filter; @@ -12,7 +11,7 @@ mod types; mod utils; type Tree = Arc>>; -type Pool = bb8::Pool>; +type Pool = sqlx::PgPool; #[derive(Debug)] pub struct Node { @@ -38,93 +37,15 @@ impl bk_tree::Metric for Hamming { async fn main() { configure_tracing(); - let s = std::env::var("POSTGRES_DSN").expect("Missing POSTGRES_DSN"); + let s = std::env::var("DATABASE_URL").expect("Missing DATABASE_URL"); - let manager = bb8_postgres::PostgresConnectionManager::new( - tokio_postgres::Config::from_str(&s).expect("Invalid POSTGRES_DSN"), - tokio_postgres::NoTls, - ); - - let db_pool = bb8::Pool::builder() - .build(manager) + let db_pool = sqlx::PgPool::connect(&s) .await - .expect("Unable to build Postgres pool"); + .expect("Unable to create Postgres pool"); let tree: Tree = Arc::new(RwLock::new(bk_tree::BKTree::new(Hamming))); - let mut max_id = 0; - - let conn = db_pool.get().await.unwrap(); - let mut lock = tree.write().await; - conn.query("SELECT id, hash FROM hashes", &[]) - .await - .unwrap() - .into_iter() - .for_each(|row| { - let id: i32 = row.get(0); - let hash: i64 = row.get(1); - let bytes = hash.to_be_bytes(); - - if id > max_id { - max_id = id; - } - - lock.add(Node { id, hash: bytes }); - }); - drop(lock); - drop(conn); - - let tree_clone = tree.clone(); - let pool_clone = db_pool.clone(); - tokio::spawn(async move { - use futures::StreamExt; - - let max_id = std::sync::atomic::AtomicI32::new(max_id); - let tree = tree_clone; - let pool = pool_clone; - - let order = std::sync::atomic::Ordering::SeqCst; - - let interval = async_stream::stream! { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); - - while let item = interval.tick().await { - yield item; - } - }; - - interval - .for_each(|_| async { - tracing::debug!("Refreshing hashes"); - - let conn = pool.get().await.unwrap(); - let mut lock = tree.write().await; - let id = max_id.load(order); - - let mut count: usize = 0; - - conn.query("SELECT id, hash FROM hashes WHERE hashes.id > $1", &[&id]) - .await - .unwrap() - .into_iter() - .for_each(|row| { - let id: i32 = row.get(0); - let hash: i64 = row.get(1); - let bytes = hash.to_be_bytes(); - - if id > max_id.load(order) { - max_id.store(id, order); - } - - lock.add(Node { id, hash: bytes }); - - count += 1; - }); - - tracing::trace!("Added {} new hashes", count); - }) - .await; - }); + load_updates(db_pool.clone(), tree.clone()).await; let log = warp::log("fuzzysearch"); let cors = warp::cors() @@ -198,6 +119,69 @@ fn configure_tracing() { registry.init(); } +#[derive(serde::Deserialize)] +struct HashRow { + id: i32, + hash: i64, +} + +async fn create_tree(conn: &Pool) -> bk_tree::BKTree { + use futures::TryStreamExt; + + let mut tree = bk_tree::BKTree::new(Hamming); + + let mut rows = sqlx::query_as!(HashRow, "SELECT id, hash FROM hashes").fetch(conn); + + while let Some(row) = rows.try_next().await.expect("Unable to get row") { + tree.add(Node { + id: row.id, + hash: row.hash.to_be_bytes(), + }) + } + + tree +} + +async fn load_updates(conn: Pool, tree: Tree) { + let mut listener = sqlx::postgres::PgListener::connect_with(&conn) + .await + .unwrap(); + listener.listen("fuzzysearch_hash_added").await.unwrap(); + + let new_tree = create_tree(&conn).await; + let mut lock = tree.write().await; + *lock = new_tree; + drop(lock); + + tokio::spawn(async move { + loop { + while let Some(notification) = listener + .try_recv() + .await + .expect("Unable to recv notification") + { + let payload: HashRow = serde_json::from_str(notification.payload()).unwrap(); + tracing::debug!(id = payload.id, "Adding new hash to tree"); + + let mut lock = tree.write().await; + lock.add(Node { + id: payload.id, + hash: payload.hash.to_be_bytes(), + }); + drop(lock); + } + + tracing::error!("Lost connection to Postgres, recreating tree"); + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + let new_tree = create_tree(&conn).await; + let mut lock = tree.write().await; + *lock = new_tree; + drop(lock); + tracing::info!("Replaced tree"); + } + }); +} + fn get_hasher() -> img_hash::Hasher<[u8; 8]> { use img_hash::{HashAlg::Gradient, HasherConfig}; diff --git a/src/models.rs b/src/models.rs index 7fc7a58..322e757 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,44 +1,31 @@ use crate::types::*; -use crate::utils::extract_rows; use crate::{Pool, Tree}; use tracing_futures::Instrument; -pub type Db<'a> = - &'a bb8::PooledConnection<'a, bb8_postgres::PostgresConnectionManager>; - #[tracing::instrument(skip(db))] -pub async fn lookup_api_key(key: &str, db: Db<'_>) -> Option { - let rows = db - .query( - "SELECT +pub async fn lookup_api_key(key: &str, db: &sqlx::PgPool) -> Option { + sqlx::query_as!( + ApiKey, + "SELECT api_key.id, api_key.name_limit, api_key.image_limit, api_key.hash_limit, api_key.name, - account.email + account.email owner_email FROM api_key JOIN account ON account.id = api_key.user_id WHERE - api_key.key = $1", - &[&key], - ) - .await - .expect("Unable to query API keys"); - - match rows.into_iter().next() { - Some(row) => Some(ApiKey { - id: row.get(0), - name_limit: row.get(1), - image_limit: row.get(2), - hash_limit: row.get(3), - name: row.get(4), - owner_email: row.get(5), - }), - _ => None, - } + api_key.key = $1 + ", + key + ) + .fetch_optional(db) + .await + .ok() + .flatten() } #[tracing::instrument(skip(pool, tree))] @@ -48,7 +35,7 @@ pub async fn image_query( hashes: Vec, distance: i64, hash: Option>, -) -> Result, tokio_postgres::Error> { +) -> Result, sqlx::Error> { let mut results = image_query_sync(pool, tree, hashes, distance, hash); let mut matches = Vec::new(); @@ -66,19 +53,26 @@ pub fn image_query_sync( hashes: Vec, distance: i64, hash: Option>, -) -> tokio::sync::mpsc::Receiver, tokio_postgres::Error>> { +) -> tokio::sync::mpsc::Receiver, sqlx::Error>> { let (tx, rx) = tokio::sync::mpsc::channel(50); tokio::spawn(async move { - let db = pool.get().await.unwrap(); + let db = pool; for query_hash in hashes { + let mut seen = std::collections::HashSet::new(); + let node = crate::Node::query(query_hash.to_be_bytes()); let lock = tree.read().await; let items = lock.find(&node, distance as u64); - for (_dist, item) in items { - let query = db.query("SELECT + for (dist, item) in items { + if seen.contains(&item.id) { + continue; + } + seen.insert(item.id); + + let row = sqlx::query!("SELECT hashes.id, hashes.hash, hashes.furaffinity_id, @@ -131,14 +125,44 @@ pub fn image_query_sync( tweet_media.hash <@ (hashes.hash, 0) LIMIT 1 ) tm ON hashes.twitter_id IS NOT NULL - WHERE hashes.id = $1", &[&item.id]).await; - let rows = query.map(|rows| { - extract_rows(rows, hash.as_deref()).into_iter().map(|mut file| { - file.searched_hash = Some(query_hash); - file - }).collect() - }); - tx.send(rows).await.unwrap(); + WHERE hashes.id = $1", item.id).map(|row| { + let (site_id, site_info) = if let Some(fa_id) = row.furaffinity_id { + ( + fa_id as i64, + Some(SiteInfo::FurAffinity(FurAffinityFile { + file_id: row.file_id.unwrap(), + })) + ) + } else if let Some(e621_id) = row.e621_id { + ( + e621_id as i64, + Some(SiteInfo::E621(E621File { + sources: row.sources, + })) + ) + } else if let Some(twitter_id) = row.twitter_id { + (twitter_id, Some(SiteInfo::Twitter)) + } else { + (-1, None) + }; + + let file = File { + id: row.id, + site_id, + site_info, + site_id_str: site_id.to_string(), + url: row.url.unwrap_or_default(), + hash: Some(row.hash), + distance: Some(dist), + artists: row.artists, + filename: row.filename.unwrap_or_default(), + searched_hash: Some(query_hash), + }; + + vec![file] + }).fetch_one(&db).await; + + tx.send(row).await.unwrap(); } } }.in_current_span()); diff --git a/src/types.rs b/src/types.rs index 55d6187..a5bd185 100644 --- a/src/types.rs +++ b/src/types.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; pub struct ApiKey { pub id: i32, pub name: Option, - pub owner_email: Option, + pub owner_email: String, pub name_limit: i16, pub image_limit: i16, pub hash_limit: i16, diff --git a/src/utils.rs b/src/utils.rs index d0e940e..ff2f4c6 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,3 @@ -use crate::models::Db; use crate::types::*; #[macro_export] @@ -51,30 +50,31 @@ macro_rules! early_return { /// joined requests. #[tracing::instrument(skip(db))] pub async fn update_rate_limit( - db: Db<'_>, + db: &sqlx::PgPool, key_id: i32, key_group_limit: i16, group_name: &'static str, incr_by: i16, -) -> Result { +) -> Result { let now = chrono::Utc::now(); let timestamp = now.timestamp(); let time_window = timestamp - (timestamp % 60); - let rows = db - .query( - "INSERT INTO - rate_limit (api_key_id, time_window, group_name, count) - VALUES - ($1, $2, $3, $4) - ON CONFLICT ON CONSTRAINT unique_window - DO UPDATE set count = rate_limit.count + $4 - RETURNING rate_limit.count", - &[&key_id, &time_window, &group_name, &incr_by], - ) - .await?; - - let count: i16 = rows[0].get(0); + let count: i16 = sqlx::query_scalar!( + "INSERT INTO + rate_limit (api_key_id, time_window, group_name, count) + VALUES + ($1, $2, $3, $4) + ON CONFLICT ON CONSTRAINT unique_window + DO UPDATE set count = rate_limit.count + $4 + RETURNING rate_limit.count", + key_id, + time_window, + group_name, + incr_by + ) + .fetch_one(db) + .await?; if count > key_group_limit { Ok(RateLimit::Limited) @@ -85,54 +85,3 @@ pub async fn update_rate_limit( ))) } } - -pub fn extract_rows( - rows: Vec, - hash: Option<&[u8]>, -) -> impl IntoIterator + '_ { - rows.into_iter().map(move |row| { - let dbhash: i64 = row.get("hash"); - let dbbytes = dbhash.to_be_bytes(); - - let (furaffinity_id, e621_id, twitter_id): (Option, Option, Option) = ( - row.get("furaffinity_id"), - row.get("e621_id"), - row.get("twitter_id"), - ); - - let (site_id, site_info) = if let Some(fa_id) = furaffinity_id { - ( - fa_id as i64, - Some(SiteInfo::FurAffinity(FurAffinityFile { - file_id: row.get("file_id"), - })), - ) - } else if let Some(e6_id) = e621_id { - ( - e6_id as i64, - Some(SiteInfo::E621(E621File { - sources: row.get("sources"), - })), - ) - } else if let Some(t_id) = twitter_id { - (t_id, Some(SiteInfo::Twitter)) - } else { - (-1, None) - }; - - File { - id: row.get("id"), - site_id, - site_info, - site_id_str: site_id.to_string(), - url: row.get("url"), - hash: Some(dbhash), - distance: hash - .map(|hash| hamming::distance_fast(&dbbytes, &hash).ok()) - .flatten(), - artists: row.get("artists"), - filename: row.get("filename"), - searched_hash: None, - } - }) -} From c345c51a0f763bf2c8f386184730de24caed1b63 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Wed, 17 Feb 2021 17:41:55 -0500 Subject: [PATCH 4/6] Add Prometheus for request and process monitoring (#4) * Add Prometheus metrics. * Set default METRICS_HOST and EXPOSE in Dockerfile. --- Cargo.lock | 51 ++++++++++++++++++++++++++++++++++++++++++++++--- Cargo.toml | 3 +++ Dockerfile | 3 ++- src/handlers.rs | 30 +++++++++++++++++++++++++++++ src/main.rs | 39 +++++++++++++++++++++++++++++++++++++ src/models.rs | 19 ++++++++++++++++++ src/utils.rs | 25 ++++++++++++++++++++++-- 7 files changed, 164 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2bfdf25..755adcd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -410,6 +410,18 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" +[[package]] +name = "flate2" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide 0.4.3", +] + [[package]] name = "fnv" version = "1.0.7" @@ -555,8 +567,10 @@ dependencies = [ "hyper", "image", "img_hash", + "lazy_static", "opentelemetry", "opentelemetry-jaeger", + "prometheus 0.11.0", "reqwest", "serde", "serde_json", @@ -1027,9 +1041,9 @@ dependencies = [ [[package]] name = "mio" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e50ae3f04d169fcc9bde0b547d1c205219b7157e07ded9c5aff03e0637cb3ed7" +checksum = "dc250d6848c90d719ea2ce34546fb5df7af1d3fd189d10bf7bad80bfcebecd95" dependencies = [ "libc", "log", @@ -1228,7 +1242,7 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project 0.4.27", - "prometheus", + "prometheus 0.7.0", "rand 0.7.3", ] @@ -1379,6 +1393,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "procfs" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8809e0c18450a2db0f236d2a44ec0b4c1412d0eb936233579f0990faa5d5cd" +dependencies = [ + "bitflags", + "byteorder", + "flate2", + "hex", + "lazy_static", + "libc", +] + [[package]] name = "prometheus" version = "0.7.0" @@ -1393,6 +1421,23 @@ dependencies = [ "spin", ] +[[package]] +name = "prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "libc", + "parking_lot", + "procfs", + "protobuf", + "regex", + "thiserror", +] + [[package]] name = "protobuf" version = "2.22.0" diff --git a/Cargo.toml b/Cargo.toml index 4181e74..30ecac9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,9 @@ tracing = "0.1" tracing-subscriber = "0.2" tracing-futures = "0.2" +prometheus = { version = "0.11", features = ["process"] } +lazy_static = "1" + opentelemetry = "0.6" opentelemetry-jaeger = "0.5" tracing-opentelemetry = "0.5" diff --git a/Dockerfile b/Dockerfile index a9cb0e5..253ff7e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,8 @@ COPY . . RUN cargo install --root / --path . FROM debian:buster-slim -EXPOSE 8080 +EXPOSE 8080 8081 +ENV METRICS_HOST=0.0.0.0:8081 WORKDIR /app RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* COPY --from=builder /bin/fuzzysearch /bin/fuzzysearch diff --git a/src/handlers.rs b/src/handlers.rs index fdbe4e9..ca64971 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -1,11 +1,31 @@ use crate::models::{image_query, image_query_sync}; use crate::types::*; use crate::{early_return, rate_limit, Pool, Tree}; +use lazy_static::lazy_static; +use prometheus::{register_histogram, register_int_counter, Histogram, IntCounter}; use std::convert::TryInto; use tracing::{span, warn}; use tracing_futures::Instrument; use warp::{Rejection, Reply}; +lazy_static! { + static ref IMAGE_HASH_DURATION: Histogram = register_histogram!( + "fuzzysearch_api_image_hash_seconds", + "Duration to perform an image hash operation" + ) + .unwrap(); + static ref IMAGE_URL_DOWNLOAD_DURATION: Histogram = register_histogram!( + "fuzzysearch_api_image_url_download_seconds", + "Duration to download an image from a provided URL" + ) + .unwrap(); + static ref UNHANDLED_REJECTIONS: IntCounter = register_int_counter!( + "fuzzysearch_api_unhandled_rejections_count", + "Number of unhandled HTTP rejections" + ) + .unwrap(); +} + #[derive(Debug)] enum Error { Postgres(sqlx::Error), @@ -87,6 +107,7 @@ async fn hash_input(form: warp::multipart::FormData) -> (i64, img_hash::ImageHas let len = bytes.len(); + let _timer = IMAGE_HASH_DURATION.start_timer(); let hash = tokio::task::spawn_blocking(move || { let hasher = crate::get_hasher(); let image = image::load_from_memory(&bytes).unwrap(); @@ -95,6 +116,7 @@ async fn hash_input(form: warp::multipart::FormData) -> (i64, img_hash::ImageHas .instrument(span!(tracing::Level::TRACE, "hashing image", len)) .await .unwrap(); + drop(_timer); let mut buf: [u8; 8] = [0; 8]; buf.copy_from_slice(&hash.as_bytes()); @@ -396,6 +418,8 @@ pub async fn search_image_by_url( let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); + let _timer = IMAGE_URL_DOWNLOAD_DURATION.start_timer(); + let mut resp = match reqwest::get(&url).await { Ok(resp) => resp, Err(_err) => return Ok(Box::new(Error::InvalidImage)), @@ -425,6 +449,9 @@ pub async fn search_image_by_url( buf.put(chunk); } + drop(_timer); + + let _timer = IMAGE_HASH_DURATION.start_timer(); let hash = tokio::task::spawn_blocking(move || { let hasher = crate::get_hasher(); let image = image::load_from_memory(&buf).unwrap(); @@ -433,6 +460,7 @@ pub async fn search_image_by_url( .instrument(span!(tracing::Level::TRACE, "hashing image")) .await .unwrap(); + drop(_timer); let hash: [u8; 8] = hash.as_bytes().try_into().unwrap(); let num = i64::from_be_bytes(hash); @@ -461,6 +489,8 @@ pub async fn search_image_by_url( pub async fn handle_rejection(err: Rejection) -> Result, std::convert::Infallible> { warn!("had rejection"); + UNHANDLED_REJECTIONS.inc(); + let (code, message) = if err.is_not_found() { ( warp::http::StatusCode::NOT_FOUND, diff --git a/src/main.rs b/src/main.rs index f005325..f23abdf 100644 --- a/src/main.rs +++ b/src/main.rs @@ -43,6 +43,8 @@ async fn main() { .await .expect("Unable to create Postgres pool"); + serve_metrics().await; + let tree: Tree = Arc::new(RwLock::new(bk_tree::BKTree::new(Hamming))); load_updates(db_pool.clone(), tree.clone()).await; @@ -119,6 +121,43 @@ fn configure_tracing() { registry.init(); } +async fn metrics( + _: hyper::Request, +) -> Result, std::convert::Infallible> { + use hyper::{Body, Response}; + use prometheus::{Encoder, TextEncoder}; + + let mut buffer = Vec::new(); + let encoder = TextEncoder::new(); + + let metric_families = prometheus::gather(); + encoder.encode(&metric_families, &mut buffer).unwrap(); + + Ok(Response::new(Body::from(buffer))) +} + +async fn serve_metrics() { + use hyper::{ + service::{make_service_fn, service_fn}, + Server, + }; + use std::convert::Infallible; + use std::net::SocketAddr; + + let make_svc = make_service_fn(|_conn| async { Ok::<_, Infallible>(service_fn(metrics)) }); + + let addr: SocketAddr = std::env::var("METRICS_HOST") + .expect("Missing METRICS_HOST") + .parse() + .expect("Invalid METRICS_HOST"); + + let server = Server::bind(&addr).serve(make_svc); + + tokio::spawn(async move { + server.await.expect("Metrics server error"); + }); +} + #[derive(serde::Deserialize)] struct HashRow { id: i32, diff --git a/src/models.rs b/src/models.rs index 322e757..e240c9e 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,7 +1,22 @@ use crate::types::*; use crate::{Pool, Tree}; +use lazy_static::lazy_static; +use prometheus::{register_histogram, Histogram}; use tracing_futures::Instrument; +lazy_static! { + static ref IMAGE_LOOKUP_DURATION: Histogram = register_histogram!( + "fuzzysearch_api_image_lookup_seconds", + "Duration to perform an image lookup" + ) + .unwrap(); + static ref IMAGE_QUERY_DURATION: Histogram = register_histogram!( + "fuzzysearch_api_image_query_seconds", + "Duration to perform a single image lookup query" + ) + .unwrap(); +} + #[tracing::instrument(skip(db))] pub async fn lookup_api_key(key: &str, db: &sqlx::PgPool) -> Option { sqlx::query_as!( @@ -62,6 +77,8 @@ pub fn image_query_sync( for query_hash in hashes { let mut seen = std::collections::HashSet::new(); + let _timer = IMAGE_LOOKUP_DURATION.start_timer(); + let node = crate::Node::query(query_hash.to_be_bytes()); let lock = tree.read().await; let items = lock.find(&node, distance as u64); @@ -72,6 +89,8 @@ pub fn image_query_sync( } seen.insert(item.id); + let _timer = IMAGE_QUERY_DURATION.start_timer(); + let row = sqlx::query!("SELECT hashes.id, hashes.hash, diff --git a/src/utils.rs b/src/utils.rs index ff2f4c6..dad259a 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,15 @@ use crate::types::*; +use lazy_static::lazy_static; +use prometheus::{register_int_counter_vec, IntCounterVec}; + +lazy_static! { + pub static ref RATE_LIMIT_STATUS: IntCounterVec = register_int_counter_vec!( + "fuzzysearch_api_rate_limit_count", + "Number of allowed and rate limited requests", + &["status"] + ) + .unwrap(); +} #[macro_export] macro_rules! rate_limit { @@ -26,8 +37,18 @@ macro_rules! rate_limit { }; match rate_limit { - crate::types::RateLimit::Limited => return Ok(Box::new(Error::RateLimit)), - crate::types::RateLimit::Available(count) => count, + crate::types::RateLimit::Limited => { + crate::utils::RATE_LIMIT_STATUS + .with_label_values(&["limited"]) + .inc(); + return Ok(Box::new(Error::RateLimit)); + } + crate::types::RateLimit::Available(count) => { + crate::utils::RATE_LIMIT_STATUS + .with_label_values(&["allowed"]) + .inc(); + count + } } }}; } From 274d54573485f433f21d5a31cf2a627c113b630d Mon Sep 17 00:00:00 2001 From: Syfaro Date: Fri, 19 Feb 2021 15:38:07 -0500 Subject: [PATCH 5/6] Include content rating. --- src/handlers.rs | 4 ++++ src/models.rs | 12 +++++++++++- src/types.rs | 25 +++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/handlers.rs b/src/handlers.rs index ca64971..9406694 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -301,6 +301,7 @@ pub async fn search_file( submission.url, submission.filename, submission.file_id, + submission.rating, artist.name, hashes.id hash_id FROM @@ -321,6 +322,7 @@ pub async fn search_file( submission.url, submission.filename, submission.file_id, + submission.rating, artist.name, hashes.id hash_id FROM @@ -341,6 +343,7 @@ pub async fn search_file( submission.url, submission.filename, submission.file_id, + submission.rating, artist.name, hashes.id hash_id FROM @@ -374,6 +377,7 @@ pub async fn search_file( file_id: row.get("file_id"), })), searched_hash: None, + rating: row.get::("rating").parse().ok(), }) .fetch_all(&db) .await; diff --git a/src/models.rs b/src/models.rs index e240c9e..2857ca9 100644 --- a/src/models.rs +++ b/src/models.rs @@ -117,7 +117,16 @@ pub fn image_query_sync( END file_id, CASE WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'sources')) - END sources + END sources, + CASE + WHEN furaffinity_id IS NOT NULL THEN (f.rating) + WHEN e621_id IS NOT NULL THEN (e.data->>'rating') + WHEN twitter_id IS NOT NULL THEN + CASE + WHEN (tw.data->'possibly_sensitive')::boolean IS true THEN 'adult' + WHEN (tw.data->'possibly_sensitive')::boolean IS false THEN 'general' + END + END rating FROM hashes LEFT JOIN LATERAL ( @@ -169,6 +178,7 @@ pub fn image_query_sync( id: row.id, site_id, site_info, + rating: row.rating.and_then(|rating| rating.parse().ok()), site_id_str: site_id.to_string(), url: row.url.unwrap_or_default(), hash: Some(row.hash), diff --git a/src/types.rs b/src/types.rs index a5bd185..3586539 100644 --- a/src/types.rs +++ b/src/types.rs @@ -23,6 +23,29 @@ pub enum RateLimit { Available((i16, i16)), } +#[derive(Debug, Serialize)] +#[serde(rename_all = "lowercase")] +pub enum Rating { + General, + Mature, + Adult, +} + +impl std::str::FromStr for Rating { + type Err = &'static str; + + fn from_str(s: &str) -> Result { + let rating = match s { + "g" | "s" | "general" => Self::General, + "m" | "q" | "mature" => Self::Mature, + "a" | "e" | "adult" => Self::Adult, + _ => return Err("unknown rating"), + }; + + Ok(rating) + } +} + /// A general type for every file. #[derive(Debug, Default, Serialize)] pub struct File { @@ -39,6 +62,8 @@ pub struct File { #[serde(flatten)] pub site_info: Option, + pub rating: Option, + #[serde(skip_serializing_if = "Option::is_none")] pub hash: Option, #[serde(skip_serializing_if = "Option::is_none")] From 6fe6c8ee0dc819524ecc63f6904ad007300ba780 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Fri, 19 Feb 2021 15:47:31 -0500 Subject: [PATCH 6/6] Fixes for content ratings. --- sqlx-data.json | 154 +++++++++++++++++++++++++----------------------- src/handlers.rs | 4 +- 2 files changed, 83 insertions(+), 75 deletions(-) diff --git a/sqlx-data.json b/sqlx-data.json index db41156..0b596c7 100644 --- a/sqlx-data.json +++ b/sqlx-data.json @@ -23,6 +23,86 @@ ] } }, + "1bd0057782de5a3b41f90081a31d24d14bb70299391050c3404742a6d2915d9e": { + "query": "SELECT\n hashes.id,\n hashes.hash,\n hashes.furaffinity_id,\n hashes.e621_id,\n hashes.twitter_id,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.url)\n WHEN e621_id IS NOT NULL THEN (e.data->'file'->>'url')\n WHEN twitter_id IS NOT NULL THEN (tm.url)\n END url,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.filename)\n WHEN e621_id IS NOT NULL THEN ((e.data->'file'->>'md5') || '.' || (e.data->'file'->>'ext'))\n WHEN twitter_id IS NOT NULL THEN (SELECT split_part(split_part(tm.url, '/', 5), ':', 1))\n END filename,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (ARRAY(SELECT f.name))\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'tags'->'artist'))\n WHEN twitter_id IS NOT NULL THEN ARRAY(SELECT tw.data->'user'->>'screen_name')\n END artists,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.file_id)\n END file_id,\n CASE\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'sources'))\n END sources,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.rating)\n WHEN e621_id IS NOT NULL THEN (e.data->>'rating')\n WHEN twitter_id IS NOT NULL THEN\n CASE\n WHEN (tw.data->'possibly_sensitive')::boolean IS true THEN 'adult'\n WHEN (tw.data->'possibly_sensitive')::boolean IS false THEN 'general'\n END\n END rating\n FROM\n hashes\n LEFT JOIN LATERAL (\n SELECT *\n FROM submission\n JOIN artist ON submission.artist_id = artist.id\n WHERE submission.id = hashes.furaffinity_id\n ) f ON hashes.furaffinity_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM e621\n WHERE e621.id = hashes.e621_id\n ) e ON hashes.e621_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet\n WHERE tweet.id = hashes.twitter_id\n ) tw ON hashes.twitter_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet_media\n WHERE\n tweet_media.tweet_id = hashes.twitter_id AND\n tweet_media.hash <@ (hashes.hash, 0)\n LIMIT 1\n ) tm ON hashes.twitter_id IS NOT NULL\n WHERE hashes.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "furaffinity_id", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "e621_id", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "twitter_id", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "url", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "filename", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "artists", + "type_info": "TextArray" + }, + { + "ordinal": 8, + "name": "file_id", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "sources", + "type_info": "TextArray" + }, + { + "ordinal": 10, + "name": "rating", + "type_info": "Bpchar" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + null, + null, + null, + null, + null, + null + ] + } + }, "659ee9ddc1c5ccd42ba9dc1617440544c30ece449ba3ba7f9d39f447b8af3cfe": { "query": "SELECT\n api_key.id,\n api_key.name_limit,\n api_key.image_limit,\n api_key.hash_limit,\n api_key.name,\n account.email owner_email\n FROM\n api_key\n JOIN account\n ON account.id = api_key.user_id\n WHERE\n api_key.key = $1\n ", "describe": { @@ -93,80 +173,6 @@ ] } }, - "f4608ccaf739d36649cdbc5297177a989cc7763006d28c97e219bb708930972a": { - "query": "SELECT\n hashes.id,\n hashes.hash,\n hashes.furaffinity_id,\n hashes.e621_id,\n hashes.twitter_id,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.url)\n WHEN e621_id IS NOT NULL THEN (e.data->'file'->>'url')\n WHEN twitter_id IS NOT NULL THEN (tm.url)\n END url,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.filename)\n WHEN e621_id IS NOT NULL THEN ((e.data->'file'->>'md5') || '.' || (e.data->'file'->>'ext'))\n WHEN twitter_id IS NOT NULL THEN (SELECT split_part(split_part(tm.url, '/', 5), ':', 1))\n END filename,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (ARRAY(SELECT f.name))\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'tags'->'artist'))\n WHEN twitter_id IS NOT NULL THEN ARRAY(SELECT tw.data->'user'->>'screen_name')\n END artists,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.file_id)\n END file_id,\n CASE\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'sources'))\n END sources\n FROM\n hashes\n LEFT JOIN LATERAL (\n SELECT *\n FROM submission\n JOIN artist ON submission.artist_id = artist.id\n WHERE submission.id = hashes.furaffinity_id\n ) f ON hashes.furaffinity_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM e621\n WHERE e621.id = hashes.e621_id\n ) e ON hashes.e621_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet\n WHERE tweet.id = hashes.twitter_id\n ) tw ON hashes.twitter_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet_media\n WHERE\n tweet_media.tweet_id = hashes.twitter_id AND\n tweet_media.hash <@ (hashes.hash, 0)\n LIMIT 1\n ) tm ON hashes.twitter_id IS NOT NULL\n WHERE hashes.id = $1", - "describe": { - "columns": [ - { - "ordinal": 0, - "name": "id", - "type_info": "Int4" - }, - { - "ordinal": 1, - "name": "hash", - "type_info": "Int8" - }, - { - "ordinal": 2, - "name": "furaffinity_id", - "type_info": "Int4" - }, - { - "ordinal": 3, - "name": "e621_id", - "type_info": "Int4" - }, - { - "ordinal": 4, - "name": "twitter_id", - "type_info": "Int8" - }, - { - "ordinal": 5, - "name": "url", - "type_info": "Text" - }, - { - "ordinal": 6, - "name": "filename", - "type_info": "Text" - }, - { - "ordinal": 7, - "name": "artists", - "type_info": "TextArray" - }, - { - "ordinal": 8, - "name": "file_id", - "type_info": "Int4" - }, - { - "ordinal": 9, - "name": "sources", - "type_info": "TextArray" - } - ], - "parameters": { - "Left": [ - "Int4" - ] - }, - "nullable": [ - false, - false, - true, - true, - true, - null, - null, - null, - null, - null - ] - } - }, "fe60be66b2d8a8f02b3bfe06d1f0e57e4bb07e80cba1b379a5f17f6cbd8b075c": { "query": "SELECT id, hash FROM hashes", "describe": { diff --git a/src/handlers.rs b/src/handlers.rs index 9406694..167e29e 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -377,7 +377,9 @@ pub async fn search_file( file_id: row.get("file_id"), })), searched_hash: None, - rating: row.get::("rating").parse().ok(), + rating: row + .get::, _>("rating") + .and_then(|rating| rating.parse().ok()), }) .fetch_all(&db) .await;