From 908cda8ce9a53ef0da6b94ba3a760ec6d58493b1 Mon Sep 17 00:00:00 2001 From: Syfaro Date: Wed, 17 Feb 2021 16:30:05 -0500 Subject: [PATCH] Use NOTIFY/LISTEN instead of polling for updates (#3) * Use NOTIFY/LISTEN instead of polling for updates. * Allow different distances for multiple hashes. --- .gitignore | 1 + Cargo.lock | 678 +++++++++++++++++++++++++++++++----------------- Cargo.toml | 4 +- Dockerfile | 1 + sqlx-data.json | 194 ++++++++++++++ src/handlers.rs | 201 +++++++------- src/main.rs | 152 +++++------ src/models.rs | 102 +++++--- src/types.rs | 2 +- src/utils.rs | 85 ++---- 10 files changed, 891 insertions(+), 529 deletions(-) create mode 100644 sqlx-data.json diff --git a/.gitignore b/.gitignore index ea8c4bf..fedaa2b 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ /target +.env diff --git a/Cargo.lock b/Cargo.lock index ee5443e..2bfdf25 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,5 +1,7 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. +version = 3 + [[package]] name = "adler" version = "0.2.3" @@ -12,6 +14,32 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "ahash" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e" + +[[package]] +name = "ahash" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "796540673305a66d127804eef19ad696f1f204b8c1025aaca4958c17eab32877" +dependencies = [ + "getrandom 0.2.2", + "once_cell", + "version_check", +] + +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -21,6 +49,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "arrayvec" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b" + [[package]] name = "async-stream" version = "0.3.0" @@ -43,14 +77,12 @@ dependencies = [ ] [[package]] -name = "async-trait" -version = "0.1.42" +name = "atoi" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" +checksum = "616896e05fc0e2649463a93a15183c6a16bf03413a7af88ef1285ddedfa9cda5" dependencies = [ - "proc-macro2", - "quote", - "syn", + "num-traits", ] [[package]] @@ -71,37 +103,24 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" -[[package]] -name = "bb8" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dae93eccab998c4b8703e3a6bbaa1714c38e445ebacb4bede25d0408521e293c" -dependencies = [ - "async-trait", - "futures-channel", - "futures-util", - "parking_lot", - "tokio", -] - -[[package]] -name = "bb8-postgres" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61fdf56d52b2cca401d2380407e5c35d3d25d3560224ecf74d6e4ca13e51239b" -dependencies = [ - "async-trait", - "bb8", - "tokio", - "tokio-postgres", -] - [[package]] name = "bitflags" version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" +[[package]] +name = "bitvec" +version = "0.19.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81" +dependencies = [ + "funty", + "radium", + "tap", + "wyz", +] + [[package]] name = "bk-tree" version = "0.3.0" @@ -149,10 +168,16 @@ dependencies = [ ] [[package]] -name = "bumpalo" -version = "3.4.0" +name = "build_const" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" +checksum = "39092a32794787acd8525ee150305ff051b0aa6cc2abaf193924f5ab05425f39" + +[[package]] +name = "bumpalo" +version = "3.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "099e596ef14349721d9016f6b80dd3419ea1bf289ab9b44df8e4dfd3a005d5d9" [[package]] name = "byte-tools" @@ -243,6 +268,15 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8aebca1129a03dc6dc2b127edd729435bbc4a37e1d5f4d7513165089ceb02634" +[[package]] +name = "crc" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d663548de7f5cca343f1e0a48d14dcfb0e9eb4e079ec58883b7251539fa10aeb" +dependencies = [ + "build_const", +] + [[package]] name = "crc32fast" version = "1.2.1" @@ -287,6 +321,16 @@ dependencies = [ "scopeguard", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f6cb3c7f5b8e51bc3ebb73a2327ad4abdbd119dc13223f14f961d2f38486756" +dependencies = [ + "cfg-if 1.0.0", + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.1" @@ -336,17 +380,26 @@ dependencies = [ "generic-array 0.14.4", ] +[[package]] +name = "dotenv" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" + [[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +dependencies = [ + "serde", +] [[package]] name = "encoding_rs" -version = "0.8.26" +version = "0.8.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +checksum = "80df024fbc5ac80f87dfef0d9f5209a252f2a497f7f42944cff24d8253cac065" dependencies = [ "cfg-if 1.0.0", ] @@ -357,12 +410,6 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e88a8acf291dafb59c2d96e8f59828f3838bb1a70398823ade51a84de6a6deed" -[[package]] -name = "fallible-iterator" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7" - [[package]] name = "fnv" version = "1.0.7" @@ -394,6 +441,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "funty" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fed34cd105917e91daa4da6b3728c47b068749d6a62c59811f06ed2ac71d9da7" + [[package]] name = "futures" version = "0.3.12" @@ -494,8 +547,6 @@ name = "fuzzysearch" version = "0.1.0" dependencies = [ "async-stream", - "bb8", - "bb8-postgres", "bk-tree", "bytes", "chrono", @@ -509,8 +560,8 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sqlx", "tokio", - "tokio-postgres", "tracing", "tracing-futures", "tracing-opentelemetry", @@ -556,7 +607,7 @@ checksum = "c9495705279e7140bf035dde1f6e750c162df8b625267cd52cc44e0b156732c8" dependencies = [ "cfg-if 1.0.0", "libc", - "wasi 0.10.1+wasi-snapshot-preview1", + "wasi 0.10.2+wasi-snapshot-preview1", ] [[package]] @@ -600,6 +651,18 @@ name = "hashbrown" version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +dependencies = [ + "ahash 0.4.7", +] + +[[package]] +name = "hashlink" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8" +dependencies = [ + "hashbrown", +] [[package]] name = "headers" @@ -626,6 +689,15 @@ dependencies = [ "http", ] +[[package]] +name = "heck" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87cbf45460356b7deeb5e3415b5563308c0a9b057c85e12b06ad551f98d0a6ac" +dependencies = [ + "unicode-segmentation", +] + [[package]] name = "hermit-abi" version = "0.1.18" @@ -635,6 +707,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "hmac" version = "0.10.1" @@ -668,9 +746,9 @@ dependencies = [ [[package]] name = "httparse" -version = "1.3.4" +version = "1.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd179ae861f0c2e53da70d892f5f3029f9594be0c41dc5269cd371691b1dc2f9" +checksum = "615caabe2c3160b313d52ccc905335f4ed5f10881dd63dc5699d47e90be85691" [[package]] name = "httpdate" @@ -680,9 +758,9 @@ checksum = "494b4d60369511e7dea41cf646832512a94e542f68bb9c49e54518e0f468eb47" [[package]] name = "hyper" -version = "0.14.2" +version = "0.14.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12219dc884514cb4a6a03737f4413c0e01c23a1b059b0156004b23f1e19dccbe" +checksum = "e8e946c2b1349055e0b72ae281b238baf1a3ea7307c7e9f9d64673bdd9c26ac7" dependencies = [ "bytes", "futures-channel", @@ -694,7 +772,7 @@ dependencies = [ "httparse", "httpdate", "itoa", - "pin-project 1.0.4", + "pin-project 1.0.5", "socket2", "tokio", "tower-service", @@ -717,9 +795,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "de910d521f7cc3135c4de8db1cb910e0b5ed1dc6f57c381cd07e8e661ce10094" dependencies = [ "matches", "unicode-bidi", @@ -728,9 +806,9 @@ dependencies = [ [[package]] name = "image" -version = "0.23.12" +version = "0.23.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ce04077ead78e39ae8610ad26216aed811996b043d47beed5090db674f9e9b5" +checksum = "293f07a1875fa7e9c5897b51aa68b2d8ed8271b87e1a44cb64b9c3d98aabbc0d" dependencies = [ "bytemuck", "byteorder", @@ -788,9 +866,9 @@ dependencies = [ [[package]] name = "integer-encoding" -version = "1.1.6" +version = "1.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6104619c35f8835695e517cfb80fb7142139ee4b53f4d0fa4c8dca6e98fbc66" +checksum = "48dc51180a9b377fd75814d0cc02199c20f8e99433d6762f650d39cdbbd3b56f" [[package]] name = "ipnet" @@ -806,19 +884,18 @@ checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "jpeg-decoder" -version = "0.1.20" +version = "0.1.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc797adac5f083b8ff0ca6f6294a999393d76e197c36488e2ef732c4715f6fa3" +checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2" dependencies = [ - "byteorder", "rayon", ] [[package]] name = "js-sys" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +checksum = "5cfb73131c35423a367daf8cbd24100af0d077668c8c2943f0e7dd775fef0f65" dependencies = [ "wasm-bindgen", ] @@ -830,10 +907,23 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] -name = "libc" -version = "0.2.82" +name = "lexical-core" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929" +checksum = "21f866863575d0e1d654fbeeabdc927292fdf862873dc3c96c6f753357e13374" +dependencies = [ + "arrayvec", + "bitflags", + "cfg-if 1.0.0", + "ryu", + "static_assertions", +] + +[[package]] +name = "libc" +version = "0.2.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7282d924be3275cec7f6756ff4121987bc6481325397dde6ba3e7802b1a8b1c" [[package]] name = "lock_api" @@ -846,13 +936,19 @@ dependencies = [ [[package]] name = "log" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcf3805d4480bb5b86070dcfeb9e2cb2ebc148adb753c5cca5f884d1d65a42b2" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", ] +[[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + [[package]] name = "matchers" version = "0.0.1" @@ -869,10 +965,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08" [[package]] -name = "md5" -version = "0.7.0" +name = "md-5" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771" +checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15" +dependencies = [ + "block-buffer 0.9.0", + "digest 0.9.0", + "opaque-debug 0.3.0", +] [[package]] name = "memchr" @@ -983,6 +1084,19 @@ dependencies = [ "tempfile", ] +[[package]] +name = "nom" +version = "6.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7413f999671bd4745a7b624bd370a569fb6bc574b23c83a3c5ed2e453f3d5e2" +dependencies = [ + "bitvec", + "funty", + "lexical-core", + "memchr", + "version_check", +] + [[package]] name = "ntapi" version = "0.3.6" @@ -1150,14 +1264,14 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ccb628cad4f84851442432c60ad8e1f607e29752d0bf072cbd0baf28aa34272" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ "cfg-if 1.0.0", "instant", "libc", - "redox_syscall 0.1.57", + "redox_syscall", "smallvec", "winapi", ] @@ -1168,24 +1282,6 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "phf" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" -dependencies = [ - "phf_shared", -] - -[[package]] -name = "phf_shared" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7" -dependencies = [ - "siphasher", -] - [[package]] name = "pin-project" version = "0.4.27" @@ -1197,11 +1293,11 @@ dependencies = [ [[package]] name = "pin-project" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95b70b68509f17aa2857863b6fa00bf21fc93674c7a8893de2f469f6aa7ca2f2" +checksum = "96fa8ebb90271c4477f144354485b8068bd8f6b78b428b01ba892ca26caf0b63" dependencies = [ - "pin-project-internal 1.0.4", + "pin-project-internal 1.0.5", ] [[package]] @@ -1217,9 +1313,9 @@ dependencies = [ [[package]] name = "pin-project-internal" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa25a6393f22ce819b0f50e0be89287292fda8d425be38ee0ca14c4931d9e71" +checksum = "758669ae3558c6f74bd2a18b41f7ac0b5a195aea6639d6a9b5e5d1ad5ba24c0b" dependencies = [ "proc-macro2", "quote", @@ -1256,35 +1352,6 @@ dependencies = [ "miniz_oxide 0.3.7", ] -[[package]] -name = "postgres-protocol" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70e34ad3dc5c56d036b9418185ee97e14b6766d55c8ccf9dc18302ad4e6371d9" -dependencies = [ - "base64 0.13.0", - "byteorder", - "bytes", - "fallible-iterator", - "hmac", - "md5", - "memchr", - "rand 0.8.2", - "sha2", - "stringprep", -] - -[[package]] -name = "postgres-types" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5493d9d4613b88b12433aa12890e74e74cd93fdc1e08b7c2aed4768aaae8414c" -dependencies = [ - "bytes", - "fallible-iterator", - "postgres-protocol", -] - [[package]] name = "ppv-lite86" version = "0.2.10" @@ -1328,9 +1395,9 @@ dependencies = [ [[package]] name = "protobuf" -version = "2.20.0" +version = "2.22.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86473d5f16580f10b131a0bf0afb68f8e029d1835d33a00f37281b05694e5312" +checksum = "73f72884896d22e0da0e5b266cb9a780b791f6c3b2f5beab6368d6cd4f0dbb86" [[package]] name = "quick-error" @@ -1340,13 +1407,19 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.8" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] +[[package]] +name = "radium" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8" + [[package]] name = "rand" version = "0.7.3" @@ -1362,13 +1435,13 @@ dependencies = [ [[package]] name = "rand" -version = "0.8.2" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "18519b42a40024d661e1714153e9ad0c3de27cd495760ceb09710920f1098b1e" +checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e" dependencies = [ "libc", "rand_chacha 0.3.0", - "rand_core 0.6.1", + "rand_core 0.6.2", "rand_hc 0.3.0", ] @@ -1389,7 +1462,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e12735cf05c9e10bf21534da50a147b924d555dc7a547c42e6bb2d5b6017ae0d" dependencies = [ "ppv-lite86", - "rand_core 0.6.1", + "rand_core 0.6.2", ] [[package]] @@ -1403,9 +1476,9 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c026d7df8b298d90ccbbc5190bd04d85e159eaf5576caeacf8741da93ccbd2e5" +checksum = "34cf66eb183df1c5876e2dcf6b13d57340741e8dc255b48e40a26de954d06ae7" dependencies = [ "getrandom 0.2.2", ] @@ -1425,7 +1498,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3190ef7066a446f2e7f42e239d161e905420ccab01eb967c9eb27d21b2322a73" dependencies = [ - "rand_core 0.6.1", + "rand_core 0.6.2", ] [[package]] @@ -1455,15 +1528,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" - -[[package]] -name = "redox_syscall" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ec8ca9416c5ea37062b502703cd7fcb207736bc294f6e0cf367ac6fc234570" +checksum = "94341e4e44e24f6b591b59e47a8a027df12e008d73fd5672dbea9cc22f4507d9" dependencies = [ "bitflags", ] @@ -1474,7 +1541,10 @@ version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9251239e129e16308e70d853559389de218ac275b515068abc96829d05b948a" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", + "thread_local", ] [[package]] @@ -1623,18 +1693,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.120" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "166b2349061381baf54a58e4b13c89369feb0ef2eaa57198899e2312aac30aab" +checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.120" +version = "1.0.123" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca2a8cb5805ce9e3b95435e3765b7b553cecc762d938d409434338386cb5775" +checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" dependencies = [ "proc-macro2", "quote", @@ -1643,10 +1713,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.61" +version = "1.0.62" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +checksum = "ea1c6153794552ea7cf7cf63b1231a25de00ec90db326ba6264440fa08e31486" dependencies = [ + "indexmap", "itoa", "ryu", "serde", @@ -1678,9 +1749,9 @@ dependencies = [ [[package]] name = "sha-1" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce3cdf1b5e620a498ee6f2a171885ac7e22f0e12089ec4b3d22b84921792507c" +checksum = "dfebf75d25bd900fd1e7d11501efab59bc846dbc76196839663e6637bba9f25f" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -1691,9 +1762,9 @@ dependencies = [ [[package]] name = "sha2" -version = "0.9.2" +version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e7aab86fe2149bad8c507606bdb3f4ef5e7b2380eb92350f56122cca72a42a8" +checksum = "fa827a14b29ab7f44778d14a88d3cb76e949c45083f7dbfa507d0cb699dc12de" dependencies = [ "block-buffer 0.9.0", "cfg-if 1.0.0", @@ -1720,12 +1791,6 @@ dependencies = [ "libc", ] -[[package]] -name = "siphasher" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa8f3741c7372e75519bd9346068370c9cdaabcc1f9599cbcf2a2719352286b7" - [[package]] name = "slab" version = "0.4.2" @@ -1755,6 +1820,116 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "sqlformat" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74c70f0235b9925cbb106c52af1a28b5ea4885a8b851e328b8562e257a389c2d" +dependencies = [ + "lazy_static", + "maplit", + "nom", + "regex", + "unicode_categories", +] + +[[package]] +name = "sqlx" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2739d54a2ae9fdd0f545cb4e4b5574efb95e2ec71b7f921678e246fb20dcaaf" +dependencies = [ + "sqlx-core", + "sqlx-macros", +] + +[[package]] +name = "sqlx-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1cad9cae4ca8947eba1a90e8ec7d3c59e7a768e2f120dc9013b669c34a90711" +dependencies = [ + "ahash 0.6.3", + "atoi", + "base64 0.13.0", + "bitflags", + "byteorder", + "bytes", + "crc", + "crossbeam-channel", + "crossbeam-queue", + "crossbeam-utils", + "either", + "futures-channel", + "futures-core", + "futures-util", + "hashlink", + "hex", + "hmac", + "itoa", + "libc", + "log", + "md-5", + "memchr", + "once_cell", + "parking_lot", + "percent-encoding", + "rand 0.7.3", + "serde", + "serde_json", + "sha-1 0.9.4", + "sha2", + "smallvec", + "sqlformat", + "sqlx-rt", + "stringprep", + "thiserror", + "tokio-stream", + "url", + "whoami", +] + +[[package]] +name = "sqlx-macros" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "01caee2b3935b4efe152f3262afbe51546ce3b1fc27ad61014e1b3cf5f55366e" +dependencies = [ + "dotenv", + "either", + "futures", + "heck", + "hex", + "once_cell", + "proc-macro2", + "quote", + "serde", + "serde_json", + "sha2", + "sqlx-core", + "sqlx-rt", + "syn", + "url", +] + +[[package]] +name = "sqlx-rt" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ce2e16b6774c671cc183e1d202386fdf9cde1e8468c1894a7f2a63eb671c4f4" +dependencies = [ + "native-tls", + "once_cell", + "tokio", + "tokio-native-tls", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "strength_reduce" version = "0.2.3" @@ -1779,15 +1954,21 @@ checksum = "1e81da0851ada1f3e9d4312c704aa4f8806f0f9d69faaf8df2f3464b4a9437c2" [[package]] name = "syn" -version = "1.0.58" +version = "1.0.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc60a3d73ea6594cd712d830cc1f0390fd71542d8c8cd24e70cc54cdfd5e05d5" +checksum = "c700597eca8a5a762beb35753ef6b94df201c81cca676604f547495a0d7f0081" dependencies = [ "proc-macro2", "quote", "unicode-xid", ] +[[package]] +name = "tap" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" + [[package]] name = "tempfile" version = "3.2.0" @@ -1796,19 +1977,39 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22" dependencies = [ "cfg-if 1.0.0", "libc", - "rand 0.8.2", - "redox_syscall 0.2.4", + "rand 0.8.3", + "redox_syscall", "remove_dir_all", "winapi", ] [[package]] -name = "thread_local" -version = "1.1.0" +name = "thiserror" +version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb9bc092d0d51e76b2b19d9d85534ffc9ec2db959a2523cdae0697e2972cd447" +checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146" dependencies = [ - "lazy_static", + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "thread_local" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8018d24e04c95ac8790716a5987d0fec4f8b27249ffa0f7d33f1369bdfb88cbd" +dependencies = [ + "once_cell", ] [[package]] @@ -1856,9 +2057,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ccf8dbc19eb42fba10e8feaaec282fb50e2c14b2726d6301dbfeed0f73306a6f" +checksum = "317cca572a0e89c3ce0ca1f1bdc9369547fe318a683418e42ac8f59d14701023" dependencies = [ "tinyvec_macros", ] @@ -1871,9 +2072,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "tokio" -version = "1.0.2" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ca04cec6ff2474c638057b65798f60ac183e5e79d3448bb7163d36a39cff6ec" +checksum = "e8190d04c665ea9e6b6a0dc45523ade572c088d2e6566244c1122671dbf4ae3a" dependencies = [ "autocfg", "bytes", @@ -1891,9 +2092,9 @@ dependencies = [ [[package]] name = "tokio-macros" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42517d2975ca3114b22a16192634e8241dc5cc1f130be194645970cc1c371494" +checksum = "caf7b11a536f46a809a8a9f0bb4237020f70ecbf115b842360afb127ea2fda57" dependencies = [ "proc-macro2", "quote", @@ -1910,34 +2111,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "tokio-postgres" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cc9f82c2bfb06a33dd0dfb44b07ca98fe72df19e681d80c78d05a1bac2138e2" -dependencies = [ - "async-trait", - "byteorder", - "bytes", - "fallible-iterator", - "futures", - "log", - "parking_lot", - "percent-encoding", - "phf", - "pin-project-lite", - "postgres-protocol", - "postgres-types", - "socket2", - "tokio", - "tokio-util", -] - [[package]] name = "tokio-stream" -version = "0.1.2" +version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76066865172052eb8796c686f0b441a93df8b08d40a950b062ffb9a426f00edd" +checksum = "1981ad97df782ab506a1f43bf82c967326960d278acf3bf8279809648c3ff3ea" dependencies = [ "futures-core", "pin-project-lite", @@ -1952,38 +2130,36 @@ checksum = "e1a5f475f1b9d077ea1017ecbc60890fda8e54942d680ca0b1d2b47cfa2d861b" dependencies = [ "futures-util", "log", - "pin-project 1.0.4", + "pin-project 1.0.5", "tokio", "tungstenite", ] [[package]] name = "tokio-util" -version = "0.6.2" +version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "feb971a26599ffd28066d387f109746df178eff14d5ea1e235015c5601967a4b" +checksum = "ebb7cb2f00c5ae8df755b252306272cd1790d39728363936e01827e11f0b017b" dependencies = [ - "async-stream", "bytes", "futures-core", "futures-sink", "log", "pin-project-lite", "tokio", - "tokio-stream", ] [[package]] name = "tower-service" -version = "0.3.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e987b6bf443f4b5b3b6f38704195592cca41c5bb7aedd3c3693c7081f8289860" +checksum = "360dfd1d6d30e05fda32ace2c8c70e9c0a9da713275777f5a4dbb8a1893930c6" [[package]] name = "tracing" -version = "0.1.22" +version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f47026cdc4080c07e49b37087de021820269d996f581aac150ef9e5583eefe3" +checksum = "f7d40a22fd029e33300d8d89a5cc8ffce18bb7c587662f54629e94c9de5487f3" dependencies = [ "cfg-if 1.0.0", "log", @@ -1994,9 +2170,9 @@ dependencies = [ [[package]] name = "tracing-attributes" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80e0ccfc3378da0cce270c946b676a376943f5cd16aeba64568e7939806f4ada" +checksum = "43f080ea7e4107844ef4766459426fa2d5c1ada2e47edba05dc7fa99d9629f47" dependencies = [ "proc-macro2", "quote", @@ -2014,11 +2190,11 @@ dependencies = [ [[package]] name = "tracing-futures" -version = "0.2.4" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab7bb6f14721aa00656086e9335d363c5c8747bae02ebe32ea2c7dece5689b4c" +checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2" dependencies = [ - "pin-project 0.4.27", + "pin-project 1.0.5", "tracing", ] @@ -2114,8 +2290,8 @@ dependencies = [ "httparse", "input_buffer", "log", - "rand 0.8.2", - "sha-1 0.9.2", + "rand 0.8.3", + "sha-1 0.9.4", "url", "utf-8", ] @@ -2155,19 +2331,31 @@ dependencies = [ [[package]] name = "unicode-normalization" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a13e63ab62dbe32aeee58d1c5408d35c36c392bba5d9d3142287219721afe606" +checksum = "07fbfce1c8a97d547e8b5334978438d9d6ec8c20e38f56d4a4374d181493eaef" dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-segmentation" +version = "1.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796" + [[package]] name = "unicode-xid" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "url" version = "2.2.0" @@ -2224,7 +2412,7 @@ dependencies = [ "mime_guess", "multipart", "percent-encoding", - "pin-project 1.0.4", + "pin-project 1.0.5", "scoped-tls", "serde", "serde_json", @@ -2246,15 +2434,15 @@ checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" [[package]] name = "wasi" -version = "0.10.1+wasi-snapshot-preview1" +version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93c6c3420963c5c64bca373b25e77acb562081b9bb4dd5bb864187742186cea9" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "wasm-bindgen" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +checksum = "55c0f7123de74f0dab9b7d00fd614e7b19349cd1e2f5252bbe9b1754b59433be" dependencies = [ "cfg-if 1.0.0", "serde", @@ -2264,9 +2452,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-backend" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +checksum = "7bc45447f0d4573f3d65720f636bbcc3dd6ce920ed704670118650bcd47764c7" dependencies = [ "bumpalo", "lazy_static", @@ -2279,9 +2467,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.19" +version = "0.4.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +checksum = "3de431a2910c86679c34283a33f66f4e4abd7e0aec27b6669060148872aadf94" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -2291,9 +2479,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +checksum = "3b8853882eef39593ad4174dd26fc9865a64e84026d223f63bb2c42affcbba2c" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2301,9 +2489,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +checksum = "4133b5e7f2a531fa413b3a1695e925038a05a71cf67e87dafa295cb645a01385" dependencies = [ "proc-macro2", "quote", @@ -2314,15 +2502,15 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.69" +version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" +checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64" [[package]] name = "web-sys" -version = "0.3.46" +version = "0.3.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +checksum = "c40dc691fc48003eba817c38da7113c15698142da971298003cac3ef175680b3" dependencies = [ "js-sys", "wasm-bindgen", @@ -2330,9 +2518,19 @@ dependencies = [ [[package]] name = "weezl" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277" +checksum = "4a32b378380f4e9869b22f0b5177c68a5519f03b3454fde0b291455ddbae266c" + +[[package]] +name = "whoami" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a921c0ad578a51c0b6c0bbb9b95f0ed11e90d61da506139e48a946edd11ee1e" +dependencies = [ + "wasm-bindgen", + "web-sys", +] [[package]] name = "winapi" @@ -2364,3 +2562,9 @@ checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" dependencies = [ "winapi", ] + +[[package]] +name = "wyz" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214" diff --git a/Cargo.toml b/Cargo.toml index 02404a6..4181e74 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,9 +28,7 @@ warp = "0.3" reqwest = "0.11" hyper = "0.14" -tokio-postgres = "0.7" -bb8 = "0.7" -bb8-postgres = "0.7" +sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "macros", "json", "offline"] } img_hash = "3" image = "0.23" diff --git a/Dockerfile b/Dockerfile index 46e231d..a9cb0e5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,6 @@ FROM rust:1-slim AS builder WORKDIR /src +ENV SQLX_OFFLINE=true RUN apt-get update -y && apt-get install -y libssl-dev pkg-config COPY . . RUN cargo install --root / --path . diff --git a/sqlx-data.json b/sqlx-data.json new file mode 100644 index 0000000..db41156 --- /dev/null +++ b/sqlx-data.json @@ -0,0 +1,194 @@ +{ + "db": "PostgreSQL", + "1984ce60f052d6a29638f8e05b35671b8edfbf273783d4b843ebd35cbb8a391f": { + "query": "INSERT INTO\n rate_limit (api_key_id, time_window, group_name, count)\n VALUES\n ($1, $2, $3, $4)\n ON CONFLICT ON CONSTRAINT unique_window\n DO UPDATE set count = rate_limit.count + $4\n RETURNING rate_limit.count", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "count", + "type_info": "Int2" + } + ], + "parameters": { + "Left": [ + "Int4", + "Int8", + "Text", + "Int2" + ] + }, + "nullable": [ + false + ] + } + }, + "659ee9ddc1c5ccd42ba9dc1617440544c30ece449ba3ba7f9d39f447b8af3cfe": { + "query": "SELECT\n api_key.id,\n api_key.name_limit,\n api_key.image_limit,\n api_key.hash_limit,\n api_key.name,\n account.email owner_email\n FROM\n api_key\n JOIN account\n ON account.id = api_key.user_id\n WHERE\n api_key.key = $1\n ", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "name_limit", + "type_info": "Int2" + }, + { + "ordinal": 2, + "name": "image_limit", + "type_info": "Int2" + }, + { + "ordinal": 3, + "name": "hash_limit", + "type_info": "Int2" + }, + { + "ordinal": 4, + "name": "name", + "type_info": "Varchar" + }, + { + "ordinal": 5, + "name": "owner_email", + "type_info": "Varchar" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + false, + false, + false, + false, + true, + false + ] + } + }, + "6b8d304fc40fa539ae671e6e24e7978ad271cb7a1cafb20fc4b4096a958d790f": { + "query": "SELECT exists(SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1))", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "exists", + "type_info": "Bool" + } + ], + "parameters": { + "Left": [ + "Text" + ] + }, + "nullable": [ + null + ] + } + }, + "f4608ccaf739d36649cdbc5297177a989cc7763006d28c97e219bb708930972a": { + "query": "SELECT\n hashes.id,\n hashes.hash,\n hashes.furaffinity_id,\n hashes.e621_id,\n hashes.twitter_id,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.url)\n WHEN e621_id IS NOT NULL THEN (e.data->'file'->>'url')\n WHEN twitter_id IS NOT NULL THEN (tm.url)\n END url,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.filename)\n WHEN e621_id IS NOT NULL THEN ((e.data->'file'->>'md5') || '.' || (e.data->'file'->>'ext'))\n WHEN twitter_id IS NOT NULL THEN (SELECT split_part(split_part(tm.url, '/', 5), ':', 1))\n END filename,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (ARRAY(SELECT f.name))\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'tags'->'artist'))\n WHEN twitter_id IS NOT NULL THEN ARRAY(SELECT tw.data->'user'->>'screen_name')\n END artists,\n CASE\n WHEN furaffinity_id IS NOT NULL THEN (f.file_id)\n END file_id,\n CASE\n WHEN e621_id IS NOT NULL THEN ARRAY(SELECT jsonb_array_elements_text(e.data->'sources'))\n END sources\n FROM\n hashes\n LEFT JOIN LATERAL (\n SELECT *\n FROM submission\n JOIN artist ON submission.artist_id = artist.id\n WHERE submission.id = hashes.furaffinity_id\n ) f ON hashes.furaffinity_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM e621\n WHERE e621.id = hashes.e621_id\n ) e ON hashes.e621_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet\n WHERE tweet.id = hashes.twitter_id\n ) tw ON hashes.twitter_id IS NOT NULL\n LEFT JOIN LATERAL (\n SELECT *\n FROM tweet_media\n WHERE\n tweet_media.tweet_id = hashes.twitter_id AND\n tweet_media.hash <@ (hashes.hash, 0)\n LIMIT 1\n ) tm ON hashes.twitter_id IS NOT NULL\n WHERE hashes.id = $1", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "hash", + "type_info": "Int8" + }, + { + "ordinal": 2, + "name": "furaffinity_id", + "type_info": "Int4" + }, + { + "ordinal": 3, + "name": "e621_id", + "type_info": "Int4" + }, + { + "ordinal": 4, + "name": "twitter_id", + "type_info": "Int8" + }, + { + "ordinal": 5, + "name": "url", + "type_info": "Text" + }, + { + "ordinal": 6, + "name": "filename", + "type_info": "Text" + }, + { + "ordinal": 7, + "name": "artists", + "type_info": "TextArray" + }, + { + "ordinal": 8, + "name": "file_id", + "type_info": "Int4" + }, + { + "ordinal": 9, + "name": "sources", + "type_info": "TextArray" + } + ], + "parameters": { + "Left": [ + "Int4" + ] + }, + "nullable": [ + false, + false, + true, + true, + true, + null, + null, + null, + null, + null + ] + } + }, + "fe60be66b2d8a8f02b3bfe06d1f0e57e4bb07e80cba1b379a5f17f6cbd8b075c": { + "query": "SELECT id, hash FROM hashes", + "describe": { + "columns": [ + { + "ordinal": 0, + "name": "id", + "type_info": "Int4" + }, + { + "ordinal": 1, + "name": "hash", + "type_info": "Int8" + } + ], + "parameters": { + "Left": [] + }, + "nullable": [ + false, + false + ] + } + } +} \ No newline at end of file diff --git a/src/handlers.rs b/src/handlers.rs index f326a51..fdbe4e9 100644 --- a/src/handlers.rs +++ b/src/handlers.rs @@ -8,8 +8,7 @@ use warp::{Rejection, Reply}; #[derive(Debug)] enum Error { - Bb8(bb8::RunError), - Postgres(tokio_postgres::Error), + Postgres(sqlx::Error), Reqwest(reqwest::Error), InvalidData, InvalidImage, @@ -20,7 +19,7 @@ enum Error { impl warp::Reply for Error { fn into_response(self) -> warp::reply::Response { let msg = match self { - Error::Bb8(_) | Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { + Error::Postgres(_) | Error::Reqwest(_) => ErrorMessage { code: 500, message: "Internal server error".to_string(), }, @@ -51,14 +50,8 @@ impl warp::Reply for Error { } } -impl From> for Error { - fn from(err: bb8::RunError) -> Self { - Error::Bb8(err) - } -} - -impl From for Error { - fn from(err: tokio_postgres::Error) -> Self { +impl From for Error { + fn from(err: sqlx::Error) -> Self { Error::Postgres(err) } } @@ -112,12 +105,10 @@ async fn hash_input(form: warp::multipart::FormData) -> (i64, img_hash::ImageHas pub async fn search_image( form: warp::multipart::FormData, opts: ImageSearchOpts, - pool: Pool, + db: Pool, tree: Tree, api_key: String, ) -> Result, Rejection> { - let db = early_return!(pool.get().await); - let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); @@ -126,7 +117,7 @@ pub async fn search_image( let mut items = { if opts.search_type == Some(ImageSearchType::Force) { image_query( - pool.clone(), + db.clone(), tree.clone(), vec![num], 10, @@ -136,7 +127,7 @@ pub async fn search_image( .unwrap() } else { let results = image_query( - pool.clone(), + db.clone(), tree.clone(), vec![num], 0, @@ -146,7 +137,7 @@ pub async fn search_image( .unwrap(); if results.is_empty() && opts.search_type != Some(ImageSearchType::Exact) { image_query( - pool.clone(), + db.clone(), tree.clone(), vec![num], 10, @@ -194,10 +185,8 @@ pub async fn stream_image( tree: Tree, api_key: String, ) -> Result, Rejection> { - let db = early_return!(pool.get().await); - - rate_limit!(&api_key, &db, image_limit, "image", 2); - rate_limit!(&api_key, &db, hash_limit, "hash"); + rate_limit!(&api_key, &pool, image_limit, "image", 2); + rate_limit!(&api_key, &pool, hash_limit, "hash"); let (num, hash) = hash_input(form).await; @@ -220,7 +209,7 @@ pub async fn stream_image( #[allow(clippy::unnecessary_wraps)] fn sse_matches( - matches: Result, tokio_postgres::Error>, + matches: Result, sqlx::Error>, ) -> Result { let items = matches.unwrap(); @@ -234,7 +223,6 @@ pub async fn search_hashes( api_key: String, ) -> Result, Rejection> { let pool = db.clone(); - let db = early_return!(db.get().await); let hashes: Vec = opts .hashes @@ -280,64 +268,95 @@ pub async fn search_file( db: Pool, api_key: String, ) -> Result, Rejection> { - let db = early_return!(db.get().await); + use sqlx::Row; let file_remaining = rate_limit!(&api_key, &db, name_limit, "file"); - let (filter, val): (&'static str, &(dyn tokio_postgres::types::ToSql + Sync)) = - if let Some(ref id) = opts.id { - ("file_id = $1", id) - } else if let Some(ref name) = opts.name { - ("lower(filename) = lower($1)", name) - } else if let Some(ref url) = opts.url { - ("lower(url) = lower($1)", url) - } else { - return Ok(Box::new(Error::InvalidData)); - }; + let query = if let Some(ref id) = opts.id { + sqlx::query( + "SELECT + submission.id, + submission.url, + submission.filename, + submission.file_id, + artist.name, + hashes.id hash_id + FROM + submission + JOIN artist + ON artist.id = submission.artist_id + JOIN hashes + ON hashes.furaffinity_id = submission.id + WHERE + file_id = $1 + LIMIT 10", + ) + .bind(id) + } else if let Some(ref name) = opts.name { + sqlx::query( + "SELECT + submission.id, + submission.url, + submission.filename, + submission.file_id, + artist.name, + hashes.id hash_id + FROM + submission + JOIN artist + ON artist.id = submission.artist_id + JOIN hashes + ON hashes.furaffinity_id = submission.id + WHERE + lower(filename) = lower($1) + LIMIT 10", + ) + .bind(name) + } else if let Some(ref url) = opts.url { + sqlx::query( + "SELECT + submission.id, + submission.url, + submission.filename, + submission.file_id, + artist.name, + hashes.id hash_id + FROM + submission + JOIN artist + ON artist.id = submission.artist_id + JOIN hashes + ON hashes.furaffinity_id = submission.id + WHERE + lower(url) = lower($1) + LIMIT 10", + ) + .bind(url) + } else { + return Ok(Box::new(Error::InvalidData)); + }; - let query = format!( - "SELECT - submission.id, - submission.url, - submission.filename, - submission.file_id, - artist.name, - hashes.id hash_id - FROM - submission - JOIN artist - ON artist.id = submission.artist_id - JOIN hashes - ON hashes.furaffinity_id = submission.id - WHERE - {} - LIMIT 10", - filter - ); + let matches: Result, _> = query + .map(|row| File { + id: row.get("hash_id"), + site_id: row.get::("id") as i64, + site_id_str: row.get::("id").to_string(), + url: row.get("url"), + filename: row.get("filename"), + artists: row + .get::, _>("name") + .map(|artist| vec![artist]), + distance: None, + hash: None, + site_info: Some(SiteInfo::FurAffinity(FurAffinityFile { + file_id: row.get("file_id"), + })), + searched_hash: None, + }) + .fetch_all(&db) + .await; - let matches: Vec<_> = early_return!( - db.query::(&*query, &[val]) - .instrument(span!(tracing::Level::TRACE, "waiting for db")) - .await - ) - .into_iter() - .map(|row| File { - id: row.get("hash_id"), - site_id: row.get::<&str, i32>("id") as i64, - site_id_str: row.get::<&str, i32>("id").to_string(), - url: row.get("url"), - filename: row.get("filename"), - artists: row - .get::<&str, Option>("name") - .map(|artist| vec![artist]), - distance: None, - hash: None, - site_info: Some(SiteInfo::FurAffinity(FurAffinityFile { - file_id: row.get("file_id"), - })), - searched_hash: None, - }) - .collect(); + let matches = early_return!(matches); let resp = warp::http::Response::builder() .header("x-rate-limit-total-file", file_remaining.1.to_string()) @@ -350,17 +369,13 @@ pub async fn search_file( } pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, Rejection> { - let db = early_return!(db.get().await); - let exists = if let Some(handle) = opts.twitter { - !early_return!( - db.query( - "SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1)", - &[&handle], - ) + let result = sqlx::query_scalar!("SELECT exists(SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1))", handle) + .fetch_optional(&db) .await - ) - .is_empty() + .map(|row| row.flatten().unwrap_or(false)); + + early_return!(result) } else { false }; @@ -370,7 +385,7 @@ pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result, pub async fn search_image_by_url( opts: UrlSearchOpts, - pool: Pool, + db: Pool, tree: Tree, api_key: String, ) -> Result, Rejection> { @@ -378,8 +393,6 @@ pub async fn search_image_by_url( let url = opts.url; - let db = early_return!(pool.get().await); - let image_remaining = rate_limit!(&api_key, &db, image_limit, "image"); let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash"); @@ -424,15 +437,9 @@ pub async fn search_image_by_url( let hash: [u8; 8] = hash.as_bytes().try_into().unwrap(); let num = i64::from_be_bytes(hash); - let results = image_query( - pool.clone(), - tree.clone(), - vec![num], - 3, - Some(hash.to_vec()), - ) - .await - .unwrap(); + let results = image_query(db.clone(), tree.clone(), vec![num], 3, Some(hash.to_vec())) + .await + .unwrap(); let resp = warp::http::Response::builder() .header("x-image-hash", num.to_string()) diff --git a/src/main.rs b/src/main.rs index a774b13..f005325 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,6 +1,5 @@ #![recursion_limit = "256"] -use std::str::FromStr; use std::sync::Arc; use tokio::sync::RwLock; use warp::Filter; @@ -12,7 +11,7 @@ mod types; mod utils; type Tree = Arc>>; -type Pool = bb8::Pool>; +type Pool = sqlx::PgPool; #[derive(Debug)] pub struct Node { @@ -38,93 +37,15 @@ impl bk_tree::Metric for Hamming { async fn main() { configure_tracing(); - let s = std::env::var("POSTGRES_DSN").expect("Missing POSTGRES_DSN"); + let s = std::env::var("DATABASE_URL").expect("Missing DATABASE_URL"); - let manager = bb8_postgres::PostgresConnectionManager::new( - tokio_postgres::Config::from_str(&s).expect("Invalid POSTGRES_DSN"), - tokio_postgres::NoTls, - ); - - let db_pool = bb8::Pool::builder() - .build(manager) + let db_pool = sqlx::PgPool::connect(&s) .await - .expect("Unable to build Postgres pool"); + .expect("Unable to create Postgres pool"); let tree: Tree = Arc::new(RwLock::new(bk_tree::BKTree::new(Hamming))); - let mut max_id = 0; - - let conn = db_pool.get().await.unwrap(); - let mut lock = tree.write().await; - conn.query("SELECT id, hash FROM hashes", &[]) - .await - .unwrap() - .into_iter() - .for_each(|row| { - let id: i32 = row.get(0); - let hash: i64 = row.get(1); - let bytes = hash.to_be_bytes(); - - if id > max_id { - max_id = id; - } - - lock.add(Node { id, hash: bytes }); - }); - drop(lock); - drop(conn); - - let tree_clone = tree.clone(); - let pool_clone = db_pool.clone(); - tokio::spawn(async move { - use futures::StreamExt; - - let max_id = std::sync::atomic::AtomicI32::new(max_id); - let tree = tree_clone; - let pool = pool_clone; - - let order = std::sync::atomic::Ordering::SeqCst; - - let interval = async_stream::stream! { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(30)); - - while let item = interval.tick().await { - yield item; - } - }; - - interval - .for_each(|_| async { - tracing::debug!("Refreshing hashes"); - - let conn = pool.get().await.unwrap(); - let mut lock = tree.write().await; - let id = max_id.load(order); - - let mut count: usize = 0; - - conn.query("SELECT id, hash FROM hashes WHERE hashes.id > $1", &[&id]) - .await - .unwrap() - .into_iter() - .for_each(|row| { - let id: i32 = row.get(0); - let hash: i64 = row.get(1); - let bytes = hash.to_be_bytes(); - - if id > max_id.load(order) { - max_id.store(id, order); - } - - lock.add(Node { id, hash: bytes }); - - count += 1; - }); - - tracing::trace!("Added {} new hashes", count); - }) - .await; - }); + load_updates(db_pool.clone(), tree.clone()).await; let log = warp::log("fuzzysearch"); let cors = warp::cors() @@ -198,6 +119,69 @@ fn configure_tracing() { registry.init(); } +#[derive(serde::Deserialize)] +struct HashRow { + id: i32, + hash: i64, +} + +async fn create_tree(conn: &Pool) -> bk_tree::BKTree { + use futures::TryStreamExt; + + let mut tree = bk_tree::BKTree::new(Hamming); + + let mut rows = sqlx::query_as!(HashRow, "SELECT id, hash FROM hashes").fetch(conn); + + while let Some(row) = rows.try_next().await.expect("Unable to get row") { + tree.add(Node { + id: row.id, + hash: row.hash.to_be_bytes(), + }) + } + + tree +} + +async fn load_updates(conn: Pool, tree: Tree) { + let mut listener = sqlx::postgres::PgListener::connect_with(&conn) + .await + .unwrap(); + listener.listen("fuzzysearch_hash_added").await.unwrap(); + + let new_tree = create_tree(&conn).await; + let mut lock = tree.write().await; + *lock = new_tree; + drop(lock); + + tokio::spawn(async move { + loop { + while let Some(notification) = listener + .try_recv() + .await + .expect("Unable to recv notification") + { + let payload: HashRow = serde_json::from_str(notification.payload()).unwrap(); + tracing::debug!(id = payload.id, "Adding new hash to tree"); + + let mut lock = tree.write().await; + lock.add(Node { + id: payload.id, + hash: payload.hash.to_be_bytes(), + }); + drop(lock); + } + + tracing::error!("Lost connection to Postgres, recreating tree"); + tokio::time::sleep(std::time::Duration::from_secs(10)).await; + let new_tree = create_tree(&conn).await; + let mut lock = tree.write().await; + *lock = new_tree; + drop(lock); + tracing::info!("Replaced tree"); + } + }); +} + fn get_hasher() -> img_hash::Hasher<[u8; 8]> { use img_hash::{HashAlg::Gradient, HasherConfig}; diff --git a/src/models.rs b/src/models.rs index 7fc7a58..322e757 100644 --- a/src/models.rs +++ b/src/models.rs @@ -1,44 +1,31 @@ use crate::types::*; -use crate::utils::extract_rows; use crate::{Pool, Tree}; use tracing_futures::Instrument; -pub type Db<'a> = - &'a bb8::PooledConnection<'a, bb8_postgres::PostgresConnectionManager>; - #[tracing::instrument(skip(db))] -pub async fn lookup_api_key(key: &str, db: Db<'_>) -> Option { - let rows = db - .query( - "SELECT +pub async fn lookup_api_key(key: &str, db: &sqlx::PgPool) -> Option { + sqlx::query_as!( + ApiKey, + "SELECT api_key.id, api_key.name_limit, api_key.image_limit, api_key.hash_limit, api_key.name, - account.email + account.email owner_email FROM api_key JOIN account ON account.id = api_key.user_id WHERE - api_key.key = $1", - &[&key], - ) - .await - .expect("Unable to query API keys"); - - match rows.into_iter().next() { - Some(row) => Some(ApiKey { - id: row.get(0), - name_limit: row.get(1), - image_limit: row.get(2), - hash_limit: row.get(3), - name: row.get(4), - owner_email: row.get(5), - }), - _ => None, - } + api_key.key = $1 + ", + key + ) + .fetch_optional(db) + .await + .ok() + .flatten() } #[tracing::instrument(skip(pool, tree))] @@ -48,7 +35,7 @@ pub async fn image_query( hashes: Vec, distance: i64, hash: Option>, -) -> Result, tokio_postgres::Error> { +) -> Result, sqlx::Error> { let mut results = image_query_sync(pool, tree, hashes, distance, hash); let mut matches = Vec::new(); @@ -66,19 +53,26 @@ pub fn image_query_sync( hashes: Vec, distance: i64, hash: Option>, -) -> tokio::sync::mpsc::Receiver, tokio_postgres::Error>> { +) -> tokio::sync::mpsc::Receiver, sqlx::Error>> { let (tx, rx) = tokio::sync::mpsc::channel(50); tokio::spawn(async move { - let db = pool.get().await.unwrap(); + let db = pool; for query_hash in hashes { + let mut seen = std::collections::HashSet::new(); + let node = crate::Node::query(query_hash.to_be_bytes()); let lock = tree.read().await; let items = lock.find(&node, distance as u64); - for (_dist, item) in items { - let query = db.query("SELECT + for (dist, item) in items { + if seen.contains(&item.id) { + continue; + } + seen.insert(item.id); + + let row = sqlx::query!("SELECT hashes.id, hashes.hash, hashes.furaffinity_id, @@ -131,14 +125,44 @@ pub fn image_query_sync( tweet_media.hash <@ (hashes.hash, 0) LIMIT 1 ) tm ON hashes.twitter_id IS NOT NULL - WHERE hashes.id = $1", &[&item.id]).await; - let rows = query.map(|rows| { - extract_rows(rows, hash.as_deref()).into_iter().map(|mut file| { - file.searched_hash = Some(query_hash); - file - }).collect() - }); - tx.send(rows).await.unwrap(); + WHERE hashes.id = $1", item.id).map(|row| { + let (site_id, site_info) = if let Some(fa_id) = row.furaffinity_id { + ( + fa_id as i64, + Some(SiteInfo::FurAffinity(FurAffinityFile { + file_id: row.file_id.unwrap(), + })) + ) + } else if let Some(e621_id) = row.e621_id { + ( + e621_id as i64, + Some(SiteInfo::E621(E621File { + sources: row.sources, + })) + ) + } else if let Some(twitter_id) = row.twitter_id { + (twitter_id, Some(SiteInfo::Twitter)) + } else { + (-1, None) + }; + + let file = File { + id: row.id, + site_id, + site_info, + site_id_str: site_id.to_string(), + url: row.url.unwrap_or_default(), + hash: Some(row.hash), + distance: Some(dist), + artists: row.artists, + filename: row.filename.unwrap_or_default(), + searched_hash: Some(query_hash), + }; + + vec![file] + }).fetch_one(&db).await; + + tx.send(row).await.unwrap(); } } }.in_current_span()); diff --git a/src/types.rs b/src/types.rs index 55d6187..a5bd185 100644 --- a/src/types.rs +++ b/src/types.rs @@ -8,7 +8,7 @@ use serde::{Deserialize, Serialize}; pub struct ApiKey { pub id: i32, pub name: Option, - pub owner_email: Option, + pub owner_email: String, pub name_limit: i16, pub image_limit: i16, pub hash_limit: i16, diff --git a/src/utils.rs b/src/utils.rs index d0e940e..ff2f4c6 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,4 +1,3 @@ -use crate::models::Db; use crate::types::*; #[macro_export] @@ -51,30 +50,31 @@ macro_rules! early_return { /// joined requests. #[tracing::instrument(skip(db))] pub async fn update_rate_limit( - db: Db<'_>, + db: &sqlx::PgPool, key_id: i32, key_group_limit: i16, group_name: &'static str, incr_by: i16, -) -> Result { +) -> Result { let now = chrono::Utc::now(); let timestamp = now.timestamp(); let time_window = timestamp - (timestamp % 60); - let rows = db - .query( - "INSERT INTO - rate_limit (api_key_id, time_window, group_name, count) - VALUES - ($1, $2, $3, $4) - ON CONFLICT ON CONSTRAINT unique_window - DO UPDATE set count = rate_limit.count + $4 - RETURNING rate_limit.count", - &[&key_id, &time_window, &group_name, &incr_by], - ) - .await?; - - let count: i16 = rows[0].get(0); + let count: i16 = sqlx::query_scalar!( + "INSERT INTO + rate_limit (api_key_id, time_window, group_name, count) + VALUES + ($1, $2, $3, $4) + ON CONFLICT ON CONSTRAINT unique_window + DO UPDATE set count = rate_limit.count + $4 + RETURNING rate_limit.count", + key_id, + time_window, + group_name, + incr_by + ) + .fetch_one(db) + .await?; if count > key_group_limit { Ok(RateLimit::Limited) @@ -85,54 +85,3 @@ pub async fn update_rate_limit( ))) } } - -pub fn extract_rows( - rows: Vec, - hash: Option<&[u8]>, -) -> impl IntoIterator + '_ { - rows.into_iter().map(move |row| { - let dbhash: i64 = row.get("hash"); - let dbbytes = dbhash.to_be_bytes(); - - let (furaffinity_id, e621_id, twitter_id): (Option, Option, Option) = ( - row.get("furaffinity_id"), - row.get("e621_id"), - row.get("twitter_id"), - ); - - let (site_id, site_info) = if let Some(fa_id) = furaffinity_id { - ( - fa_id as i64, - Some(SiteInfo::FurAffinity(FurAffinityFile { - file_id: row.get("file_id"), - })), - ) - } else if let Some(e6_id) = e621_id { - ( - e6_id as i64, - Some(SiteInfo::E621(E621File { - sources: row.get("sources"), - })), - ) - } else if let Some(t_id) = twitter_id { - (t_id, Some(SiteInfo::Twitter)) - } else { - (-1, None) - }; - - File { - id: row.get("id"), - site_id, - site_info, - site_id_str: site_id.to_string(), - url: row.get("url"), - hash: Some(dbhash), - distance: hash - .map(|hash| hamming::distance_fast(&dbbytes, &hash).ok()) - .flatten(), - artists: row.get("artists"), - filename: row.get("filename"), - searched_hash: None, - } - }) -}