anthrove/fuzzysearch
4
0
Fork 0
mirror of https://github.com/Syfaro/fuzzysearch.git synced 2024-11-23 15:22:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Rework fuzzysearch-api with Poem and OpenAPI.

Browse Source
This commit is contained in:
Syfaro 2022-01-04 00:42:22 -05:00
parent 0d650cff53
commit 5e76d4ecf2
13 changed files with 1106 additions and 1195 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

539
Cargo.lock generated
View File

@ -2,6 +2,16 @@
# It is not intended for manual editing. # It is not intended for manual editing.
version = 3 version = 3
[[package]]
name = "Inflector"
version = "0.11.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3"
dependencies = [
"lazy_static",
"regex",
]
[[package]] [[package]]
name = "actix-codec" name = "actix-codec"
version = "0.4.2" version = "0.4.2"
@ -9,7 +19,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a36c014a3e811624313b51a227b775ecba55d36ef9462bbaac7d4f13e54c9271" checksum = "a36c014a3e811624313b51a227b775ecba55d36ef9462bbaac7d4f13e54c9271"
dependencies = [ dependencies = [
"bitflags", "bitflags",
"bytes", "bytes 1.1.0",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"log", "log",
@ -30,10 +40,10 @@ dependencies = [
"actix-service", "actix-service",
"actix-utils", "actix-utils",
"ahash", "ahash",
"base64", "base64 0.13.0",
"bitflags", "bitflags",
"brotli2", "brotli2",
"bytes", "bytes 1.1.0",
"bytestring", "bytestring",
"derive_more", "derive_more",
"encoding_rs", "encoding_rs",
@ -74,14 +84,14 @@ checksum = "473ef32141eec0050af83ce8945b5e62b1609a68ce19f753a8dac1ccce8d2dd3"
dependencies = [ dependencies = [
"actix-utils", "actix-utils",
"actix-web", "actix-web",
"bytes", "bytes 1.1.0",
"derive_more", "derive_more",
"futures-core", "futures-core",
"httparse", "httparse",
"local-waker", "local-waker",
"log", "log",
"mime", "mime",
"twoway 0.2.2", "twoway",
] ]
[[package]] [[package]]
@ -164,7 +174,7 @@ dependencies = [
"actix-utils", "actix-utils",
"actix-web-codegen", "actix-web-codegen",
"ahash", "ahash",
"bytes", "bytes 1.1.0",
"cfg-if", "cfg-if",
"cookie", "cookie",
"derive_more", "derive_more",
@ -220,6 +230,41 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "aead"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0b613b8e1e3cf911a086f53f03bf286f52fd7a7258e4fa606f0ef220d39d8877"
dependencies = [
"generic-array",
]
[[package]]
name = "aes"
version = "0.7.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e8b47f52ea9bae42228d07ec09eb676433d7c4ed1ebdf0f1d1c29ed446f1ab8"
dependencies = [
"cfg-if",
"cipher",
"cpufeatures",
"opaque-debug",
]
[[package]]
name = "aes-gcm"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "df5f85a83a7d8b0442b6aa7b504b8212c1733da07b98aae43d4bc21b2cb3cdf6"
dependencies = [
"aead",
"aes",
"cipher",
"ctr",
"ghash",
"subtle",
]
[[package]] [[package]]
name = "ahash" name = "ahash"
version = "0.7.6" version = "0.7.6"
@ -302,6 +347,12 @@ dependencies = [
"rustc-demangle", "rustc-demangle",
] ]
[[package]]
name = "base64"
version = "0.11.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7"
[[package]] [[package]]
name = "base64" name = "base64"
version = "0.13.0" version = "0.13.0"
@ -367,16 +418,6 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "buf_redux"
version = "0.8.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b953a6887648bb07a535631f2bc00fbdb2a2216f135552cb3f534ed136b9c07f"
dependencies = [
"memchr",
"safemem",
]
[[package]] [[package]]
name = "bufstream" name = "bufstream"
version = "0.1.4" version = "0.1.4"
@ -401,6 +442,12 @@ version = "1.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"
[[package]]
name = "bytes"
version = "0.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38"
[[package]] [[package]]
name = "bytes" name = "bytes"
version = "1.1.0" version = "1.1.0"
@ -413,7 +460,7 @@ version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d" checksum = "90706ba19e97b90786e19dc0d5e2abd80008d99d4c0c5d1ad0b5e72cec7c494d"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
] ]
[[package]] [[package]]
@ -445,6 +492,15 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "cipher"
version = "0.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ee52072ec15386f770805afd189a01c8841be8696bed250fa2f13c4c0d6dfb7"
dependencies = [
"generic-array",
]
[[package]] [[package]]
name = "color_quant" name = "color_quant"
version = "1.1.0" version = "1.1.0"
@ -463,7 +519,14 @@ version = "0.16.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05" checksum = "94d4706de1b0fa5b132270cddffa8585166037822e260a944fe161acd137ca05"
dependencies = [ dependencies = [
"aes-gcm",
"base64 0.13.0",
"hkdf",
"hmac 0.12.0",
"percent-encoding", "percent-encoding",
"rand 0.8.4",
"sha2 0.10.0",
"subtle",
"time 0.3.5", "time 0.3.5",
"version_check", "version_check",
] ]
@ -617,6 +680,60 @@ dependencies = [
"syn", "syn",
] ]
[[package]]
name = "ctr"
version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "049bb91fb4aaf0e3c7efa6cd5ef877dbbbd15b39dad06d9948de4ec8a75761ea"
dependencies = [
"cipher",
]
[[package]]
name = "darling"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d0d720b8683f8dd83c65155f0530560cba68cd2bf395f6513a483caee57ff7f4"
dependencies = [
"darling_core",
"darling_macro",
]
[[package]]
name = "darling_core"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a340f241d2ceed1deb47ae36c4144b2707ec7dd0b649f894cb39bb595986324"
dependencies = [
"fnv",
"ident_case",
"proc-macro2",
"quote",
"strsim",
"syn",
]
[[package]]
name = "darling_macro"
version = "0.13.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "72c41b3b7352feb3211a0d743dc5700a4e3b60f51bd2b368892d1e0f9a95f44b"
dependencies = [
"darling_core",
"quote",
"syn",
]
[[package]]
name = "dashmap"
version = "4.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e77a43b28d0668df09411cb0bc9a8c2adc40f9a048afe863e05fd43251e8e39c"
dependencies = [
"cfg-if",
"num_cpus",
]
[[package]] [[package]]
name = "deflate" name = "deflate"
version = "0.8.6" version = "0.8.6"
@ -966,7 +1083,7 @@ name = "fuzzysearch-api"
version = "0.2.0" version = "0.2.0"
dependencies = [ dependencies = [
"bkapi-client", "bkapi-client",
"bytes", "bytes 1.1.0",
"chrono", "chrono",
"futures", "futures",
"fuzzysearch-common", "fuzzysearch-common",
@ -978,19 +1095,21 @@ dependencies = [
"opentelemetry", "opentelemetry",
"opentelemetry-http", "opentelemetry-http",
"opentelemetry-jaeger", "opentelemetry-jaeger",
"prometheus", "poem",
"poem-openapi",
"prometheus 0.13.0",
"reqwest", "reqwest",
"serde", "serde",
"serde_json", "serde_json",
"sqlx", "sqlx",
"thiserror",
"tokio", "tokio",
"tokio-stream", "tokio-util",
"tracing", "tracing",
"tracing-futures", "tracing-futures",
"tracing-log", "tracing-log",
"tracing-opentelemetry", "tracing-opentelemetry",
"tracing-subscriber", "tracing-subscriber",
"warp",
] ]
[[package]] [[package]]
@ -998,7 +1117,7 @@ name = "fuzzysearch-common"
version = "0.1.0" version = "0.1.0"
dependencies = [ dependencies = [
"anyhow", "anyhow",
"base64", "base64 0.13.0",
"chrono", "chrono",
"faktory", "faktory",
"futures", "futures",
@ -1009,7 +1128,7 @@ dependencies = [
"opentelemetry", "opentelemetry",
"opentelemetry-http", "opentelemetry-http",
"opentelemetry-jaeger", "opentelemetry-jaeger",
"prometheus", "prometheus 0.13.0",
"reqwest", "reqwest",
"serde", "serde",
"serde_json", "serde_json",
@ -1032,7 +1151,7 @@ dependencies = [
"fuzzysearch-common", "fuzzysearch-common",
"image", "image",
"lazy_static", "lazy_static",
"prometheus", "prometheus 0.13.0",
"tempfile", "tempfile",
"tokio", "tokio",
"tokio-stream", "tokio-stream",
@ -1050,7 +1169,7 @@ dependencies = [
"image", "image",
"img_hash", "img_hash",
"lazy_static", "lazy_static",
"prometheus", "prometheus 0.13.0",
"reqwest", "reqwest",
"serde", "serde",
"serde_json", "serde_json",
@ -1073,7 +1192,7 @@ dependencies = [
"fuzzysearch-common", "fuzzysearch-common",
"hyper", "hyper",
"lazy_static", "lazy_static",
"prometheus", "prometheus 0.13.0",
"reqwest", "reqwest",
"serde", "serde",
"serde_json", "serde_json",
@ -1092,7 +1211,7 @@ dependencies = [
"image", "image",
"img_hash", "img_hash",
"lazy_static", "lazy_static",
"prometheus", "prometheus 0.13.0",
"reqwest", "reqwest",
"serde", "serde",
"serde_json", "serde_json",
@ -1188,6 +1307,16 @@ dependencies = [
"wasi 0.10.2+wasi-snapshot-preview1", "wasi 0.10.2+wasi-snapshot-preview1",
] ]
[[package]]
name = "ghash"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1583cc1656d7839fd3732b80cf4f38850336cdb9b8ded1cd399ca62958de3c99"
dependencies = [
"opaque-debug",
"polyval",
]
[[package]] [[package]]
name = "gif" name = "gif"
version = "0.11.3" version = "0.11.3"
@ -1210,7 +1339,7 @@ version = "0.3.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd" checksum = "8f072413d126e57991455e0a922b31e4c8ba7c2ffbebf6b78b4f8521397d65cd"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"fnv", "fnv",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
@ -1253,9 +1382,9 @@ version = "0.3.5"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0" checksum = "a4c4eb0471fcb85846d8b0690695ef354f9afb11cb03cac2e1d7c9253351afb0"
dependencies = [ dependencies = [
"base64", "base64 0.13.0",
"bitflags", "bitflags",
"bytes", "bytes 1.1.0",
"headers-core", "headers-core",
"http", "http",
"httpdate", "httpdate",
@ -1296,6 +1425,15 @@ version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
[[package]]
name = "hkdf"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "94f41e9c77b6fc05b57497b960aad55942a9bbc5b20e1e623cf7fb1868f695d1"
dependencies = [
"hmac 0.12.0",
]
[[package]] [[package]]
name = "hmac" name = "hmac"
version = "0.11.0" version = "0.11.0"
@ -1346,7 +1484,7 @@ version = "0.2.6"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03" checksum = "31f4c6746584866f0feabcc69893c5b51beef3831656a968ed7ae254cdc4fd03"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"fnv", "fnv",
"itoa 1.0.1", "itoa 1.0.1",
] ]
@ -1357,7 +1495,7 @@ version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6" checksum = "1ff4f84919677303da5f147645dbea6b1881f368d03ac84e1dc09031ebd7b2c6"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"http", "http",
"pin-project-lite", "pin-project-lite",
] ]
@ -1380,7 +1518,7 @@ version = "0.14.16"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55" checksum = "b7ec3e62bdc98a2f0393a5048e4c30ef659440ea6e0e572965103e72bd836f55"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"futures-channel", "futures-channel",
"futures-core", "futures-core",
"futures-util", "futures-util",
@ -1404,13 +1542,19 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905" checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"hyper", "hyper",
"native-tls", "native-tls",
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
] ]
[[package]]
name = "ident_case"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]] [[package]]
name = "idna" name = "idna"
version = "0.2.3" version = "0.2.3"
@ -1447,7 +1591,7 @@ version = "3.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5ea4eac6fc4f64ed363d5c210732b747bfa5ddd8a25ac347d887f298c3a70b49" checksum = "5ea4eac6fc4f64ed363d5c210732b747bfa5ddd8a25ac347d887f298c3a70b49"
dependencies = [ dependencies = [
"base64", "base64 0.13.0",
"image", "image",
"rustdct", "rustdct",
"serde", "serde",
@ -1740,21 +1884,22 @@ dependencies = [
] ]
[[package]] [[package]]
name = "multipart" name = "multer"
version = "0.18.0" version = "2.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "00dec633863867f29cb39df64a397cdf4a6354708ddd7759f70c7fb51c5f9182" checksum = "5f8f35e687561d5c1667590911e6698a8cb714a134a7505718a182e7bc9d3836"
dependencies = [ dependencies = [
"buf_redux", "bytes 1.1.0",
"encoding_rs",
"futures-util",
"http",
"httparse", "httparse",
"log", "log",
"memchr",
"mime", "mime",
"mime_guess", "spin",
"quick-error", "tokio",
"rand 0.8.4", "version_check",
"safemem",
"tempfile",
"twoway 0.1.8",
] ]
[[package]] [[package]]
@ -1930,6 +2075,8 @@ checksum = "e1cf9b1c4e9a6c4de793c632496fa490bdc0e1eea73f0c91394f7b6990935d22"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"crossbeam-channel", "crossbeam-channel",
"dashmap",
"fnv",
"futures", "futures",
"js-sys", "js-sys",
"lazy_static", "lazy_static",
@ -1948,7 +2095,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d50ceb0b0e8b75cb3e388a2571a807c8228dabc5d6670f317b6eb21301095373" checksum = "d50ceb0b0e8b75cb3e388a2571a807c8228dabc5d6670f317b6eb21301095373"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"bytes", "bytes 1.1.0",
"futures-util", "futures-util",
"http", "http",
"opentelemetry", "opentelemetry",
@ -1969,6 +2116,17 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "opentelemetry-prometheus"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee9c06c1366665e7d4dba6540a42ea48900a9c92dc5b963f3ae05fbba76dc63"
dependencies = [
"opentelemetry",
"prometheus 0.12.0",
"protobuf",
]
[[package]] [[package]]
name = "opentelemetry-semantic-conventions" name = "opentelemetry-semantic-conventions"
version = "0.8.0" version = "0.8.0"
@ -2146,13 +2304,120 @@ dependencies = [
"miniz_oxide 0.3.7", "miniz_oxide 0.3.7",
] ]
[[package]]
name = "poem"
version = "1.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "729b68ea9a102b96d2d3a46954cd108bbe14e1d906b722822b0deff7d7617c3d"
dependencies = [
"async-trait",
"bytes 1.1.0",
"chrono",
"cookie",
"futures-util",
"headers",
"http",
"hyper",
"mime",
"mime_guess",
"multer",
"opentelemetry",
"opentelemetry-http",
"opentelemetry-prometheus",
"opentelemetry-semantic-conventions",
"parking_lot",
"percent-encoding",
"pin-project-lite",
"poem-derive",
"prometheus 0.12.0",
"regex",
"serde",
"serde_json",
"serde_urlencoded",
"smallvec",
"tempfile",
"thiserror",
"time 0.3.5",
"tokio",
"tokio-stream",
"tokio-util",
"tracing",
]
[[package]]
name = "poem-derive"
version = "1.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c4648e2d2f0ef5a50119594205afdceb40ef1b6ec57bfa2f949df4aaa6f8bc7"
dependencies = [
"proc-macro-crate",
"proc-macro2",
"quote",
"syn",
]
[[package]]
name = "poem-openapi"
version = "1.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "964b896103226ad7cf87a82328e1b151c1a6319af59496a23f3ef28adcc8a0f5"
dependencies = [
"base64 0.13.0",
"bytes 1.1.0",
"chrono",
"derive_more",
"futures-util",
"mime",
"num-traits",
"once_cell",
"poem",
"poem-openapi-derive",
"regex",
"serde",
"serde_json",
"thiserror",
"tokio",
"typed-headers",
]
[[package]]
name = "poem-openapi-derive"
version = "1.2.22"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "19aecdb1b13e895f34e28b078f4ef7605a7e1bae9403a5025688200ec7496f32"
dependencies = [
"Inflector",
"darling",
"http",
"indexmap",
"mime",
"proc-macro-crate",
"proc-macro2",
"quote",
"regex",
"syn",
"thiserror",
]
[[package]]
name = "polyval"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8419d2b623c7c0896ff2d5d96e2cb4ede590fed28fcc34934f4c33c036e620a1"
dependencies = [
"cfg-if",
"cpufeatures",
"opaque-debug",
"universal-hash",
]
[[package]] [[package]]
name = "postgres" name = "postgres"
version = "0.19.2" version = "0.19.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb76d6535496f633fa799bb872ffb4790e9cbdedda9d35564ca0252f930c0dd5" checksum = "eb76d6535496f633fa799bb872ffb4790e9cbdedda9d35564ca0252f930c0dd5"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"fallible-iterator", "fallible-iterator",
"futures", "futures",
"log", "log",
@ -2166,9 +2431,9 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "79ec03bce71f18b4a27c4c64c6ba2ddf74686d69b91d8714fb32ead3adaed713" checksum = "79ec03bce71f18b4a27c4c64c6ba2ddf74686d69b91d8714fb32ead3adaed713"
dependencies = [ dependencies = [
"base64", "base64 0.13.0",
"byteorder", "byteorder",
"bytes", "bytes 1.1.0",
"fallible-iterator", "fallible-iterator",
"hmac 0.12.0", "hmac 0.12.0",
"md-5 0.10.0", "md-5 0.10.0",
@ -2184,7 +2449,7 @@ version = "0.2.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d" checksum = "04619f94ba0cc80999f4fc7073607cb825bc739a883cb6d20900fc5e009d6b0d"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"chrono", "chrono",
"fallible-iterator", "fallible-iterator",
"postgres-protocol", "postgres-protocol",
@ -2202,6 +2467,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
[[package]]
name = "proc-macro-crate"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ebace6889caf889b4d3f76becee12e90353f2b8c7d875534a71e5742f8f6f83"
dependencies = [
"thiserror",
"toml",
]
[[package]] [[package]]
name = "proc-macro-hack" name = "proc-macro-hack"
version = "0.5.19" version = "0.5.19"
@ -2231,6 +2506,21 @@ dependencies = [
"libc", "libc",
] ]
[[package]]
name = "prometheus"
version = "0.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5986aa8d62380092d2f50f8b1cdba9cb9b6731ffd4b25b51fd126b6c3e05b99c"
dependencies = [
"cfg-if",
"fnv",
"lazy_static",
"memchr",
"parking_lot",
"protobuf",
"thiserror",
]
[[package]] [[package]]
name = "prometheus" name = "prometheus"
version = "0.13.0" version = "0.13.0"
@ -2254,12 +2544,6 @@ version = "2.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "47c327e191621a2158159df97cdbc2e7074bb4e940275e35abf38eb3d2595754" checksum = "47c327e191621a2158159df97cdbc2e7074bb4e940275e35abf38eb3d2595754"
[[package]]
name = "quick-error"
version = "1.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0"
[[package]] [[package]]
name = "quote" name = "quote"
version = "1.0.14" version = "1.0.14"
@ -2466,8 +2750,8 @@ version = "0.11.8"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7c4e0a76dc12a116108933f6301b95e83634e0c47b0afbed6abbaa0601e99258" checksum = "7c4e0a76dc12a116108933f6301b95e83634e0c47b0afbed6abbaa0601e99258"
dependencies = [ dependencies = [
"base64", "base64 0.13.0",
"bytes", "bytes 1.1.0",
"encoding_rs", "encoding_rs",
"futures-core", "futures-core",
"futures-util", "futures-util",
@ -2489,6 +2773,7 @@ dependencies = [
"serde_urlencoded", "serde_urlencoded",
"tokio", "tokio",
"tokio-native-tls", "tokio-native-tls",
"tokio-util",
"url", "url",
"wasm-bindgen", "wasm-bindgen",
"wasm-bindgen-futures", "wasm-bindgen-futures",
@ -2539,12 +2824,6 @@ version = "1.0.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f"
[[package]]
name = "safemem"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072"
[[package]] [[package]]
name = "schannel" name = "schannel"
version = "0.1.19" version = "0.1.19"
@ -2564,12 +2843,6 @@ dependencies = [
"parking_lot", "parking_lot",
] ]
[[package]]
name = "scoped-tls"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2"
[[package]] [[package]]
name = "scoped_threadpool" name = "scoped_threadpool"
version = "0.1.9" version = "0.1.9"
@ -2795,6 +3068,12 @@ dependencies = [
"winapi", "winapi",
] ]
[[package]]
name = "spin"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "511254be0c5bcf062b019a6c89c01a664aa359ded62f78aa72c6fc137c0590e5"
[[package]] [[package]]
name = "sqlformat" name = "sqlformat"
version = "0.1.8" version = "0.1.8"
@ -2824,10 +3103,10 @@ checksum = "518be6f6fff5ca76f985d434f9c37f3662af279642acf730388f271dff7b9016"
dependencies = [ dependencies = [
"ahash", "ahash",
"atoi", "atoi",
"base64", "base64 0.13.0",
"bitflags", "bitflags",
"byteorder", "byteorder",
"bytes", "bytes 1.1.0",
"chrono", "chrono",
"crc", "crc",
"crossbeam-channel", "crossbeam-channel",
@ -2948,6 +3227,12 @@ dependencies = [
"unicode-normalization", "unicode-normalization",
] ]
[[package]]
name = "strsim"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]] [[package]]
name = "subtle" name = "subtle"
version = "2.4.1" version = "2.4.1"
@ -3118,7 +3403,7 @@ version = "1.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838" checksum = "fbbf1c778ec206785635ce8ad57fe52b3009ae9e0c9f574a728f3049d3e55838"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"libc", "libc",
"memchr", "memchr",
"mio 0.7.14", "mio 0.7.14",
@ -3160,7 +3445,7 @@ checksum = "4b6c8b33df661b548dcd8f9bf87debb8c56c05657ed291122e1188698c2ece95"
dependencies = [ dependencies = [
"async-trait", "async-trait",
"byteorder", "byteorder",
"bytes", "bytes 1.1.0",
"fallible-iterator", "fallible-iterator",
"futures", "futures",
"log", "log",
@ -3186,26 +3471,13 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "tokio-tungstenite"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "511de3f85caf1c98983545490c3d09685fa8eb634e57eec22bb4db271f46cbd8"
dependencies = [
"futures-util",
"log",
"pin-project",
"tokio",
"tungstenite",
]
[[package]] [[package]]
name = "tokio-util" name = "tokio-util"
version = "0.6.9" version = "0.6.9"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0" checksum = "9e99e1983e5d376cd8eb4b66604d2e99e79f5bd988c3055891dcd8c9e2604cc0"
dependencies = [ dependencies = [
"bytes", "bytes 1.1.0",
"futures-core", "futures-core",
"futures-sink", "futures-sink",
"log", "log",
@ -3213,6 +3485,15 @@ dependencies = [
"tokio", "tokio",
] ]
[[package]]
name = "toml"
version = "0.5.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31142970826733df8241ef35dc040ef98c679ab14d7c3e54d827099b3acecaa"
dependencies = [
"serde",
]
[[package]] [[package]]
name = "tower-service" name = "tower-service"
version = "0.3.1" version = "0.3.1"
@ -3226,7 +3507,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105" checksum = "375a639232caf30edfc78e8d89b2d4c375515393e7af7e16f01cd96917fb2105"
dependencies = [ dependencies = [
"cfg-if", "cfg-if",
"log",
"pin-project-lite", "pin-project-lite",
"tracing-attributes", "tracing-attributes",
"tracing-core", "tracing-core",
@ -3364,34 +3644,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
[[package]]
name = "tungstenite"
version = "0.14.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a0b2d8558abd2e276b0a8df5c05a2ec762609344191e5fd23e292c910e9165b5"
dependencies = [
"base64",
"byteorder",
"bytes",
"http",
"httparse",
"log",
"rand 0.8.4",
"sha-1 0.9.8",
"thiserror",
"url",
"utf-8",
]
[[package]]
name = "twoway"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59b11b2b5241ba34be09c3cc85a36e56e48f9888862e19cedf23336d35316ed1"
dependencies = [
"memchr",
]
[[package]] [[package]]
name = "twoway" name = "twoway"
version = "0.2.2" version = "0.2.2"
@ -3402,6 +3654,19 @@ dependencies = [
"unchecked-index", "unchecked-index",
] ]
[[package]]
name = "typed-headers"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3179a61e9eccceead5f1574fd173cf2e162ac42638b9bf214c6ad0baf7efa24a"
dependencies = [
"base64 0.11.0",
"bytes 0.5.6",
"chrono",
"http",
"mime",
]
[[package]] [[package]]
name = "typenum" name = "typenum"
version = "1.15.0" version = "1.15.0"
@ -3462,6 +3727,16 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index" source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
[[package]]
name = "universal-hash"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f214e8f697e925001e66ec2c6e37a4ef93f0f78c2eed7814394e10c62025b05"
dependencies = [
"generic-array",
"subtle",
]
[[package]] [[package]]
name = "url" name = "url"
version = "2.2.2" version = "2.2.2"
@ -3511,36 +3786,6 @@ dependencies = [
"try-lock", "try-lock",
] ]
[[package]]
name = "warp"
version = "0.3.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cef4e1e9114a4b7f1ac799f16ce71c14de5778500c5450ec6b7b920c55b587e"
dependencies = [
"bytes",
"futures-channel",
"futures-util",
"headers",
"http",
"hyper",
"log",
"mime",
"mime_guess",
"multipart",
"percent-encoding",
"pin-project",
"scoped-tls",
"serde",
"serde_json",
"serde_urlencoded",
"tokio",
"tokio-stream",
"tokio-tungstenite",
"tokio-util",
"tower-service",
"tracing",
]
[[package]] [[package]]
name = "wasi" name = "wasi"
version = "0.9.0+wasi-snapshot-preview1" version = "0.9.0+wasi-snapshot-preview1"

10
fuzzysearch-api/Cargo.toml
View File

@ -10,6 +10,8 @@ tracing-subscriber = "0.3"
tracing-futures = "0.2" tracing-futures = "0.2"
tracing-log = "0.1" tracing-log = "0.1"
thiserror = "1"
prometheus = { version = "0.13", features = ["process"] } prometheus = { version = "0.13", features = ["process"] }
lazy_static = "1" lazy_static = "1"
@ -19,7 +21,7 @@ tracing-opentelemetry = "0.16"
opentelemetry-http = "0.5" opentelemetry-http = "0.5"
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }
tokio-stream = "0.1" tokio-util = "0.6"
futures = "0.3" futures = "0.3"
@ -29,8 +31,10 @@ bytes = "1"
serde = { version = "1", features = ["derive"] } serde = { version = "1", features = ["derive"] }
serde_json = "1" serde_json = "1"
warp = "0.3" poem = { version = "1", features = ["multipart", "opentelemetry", "prometheus"] }
reqwest = { version = "0.11", features = ["multipart"] } poem-openapi = { version = "1", features = ["chrono", "swagger-ui"] }
reqwest = { version = "0.11", features = ["multipart", "stream"] }
hyper = "0.14" hyper = "0.14"
sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "macros", "json", "offline", "chrono"] } sqlx = { version = "0.5", features = ["runtime-tokio-native-tls", "postgres", "macros", "json", "offline", "chrono"] }

9
fuzzysearch-api/queries/handle_twitter.sql Normal file
View File

@ -0,0 +1,9 @@
SELECT
exists(
SELECT
1
FROM
twitter_user
WHERE
lower(data ->> 'screen_name') = lower($1)
) "exists!";

12
fuzzysearch-api/queries/lookup_api_key.sql Normal file
View File

@ -0,0 +1,12 @@
SELECT
api_key.id,
api_key.name_limit,
api_key.image_limit,
api_key.hash_limit,
api_key.name,
account.email owner_email
FROM
api_key
JOIN account ON account.id = api_key.user_id
WHERE
api_key.key = $1

16
fuzzysearch-api/queries/lookup_furaffinity_file_id.sql Normal file
View File

@ -0,0 +1,16 @@
SELECT
submission.id,
submission.url,
submission.filename,
submission.file_id,
submission.rating,
submission.posted_at,
submission.hash_int hash,
artist.name artist
FROM
submission
LEFT JOIN artist ON artist.id = submission.artist_id
WHERE
file_id = $1
LIMIT
10;

76
fuzzysearch-api/queries/lookup_hashes.sql Normal file
View File

@ -0,0 +1,76 @@
WITH hashes AS (
SELECT * FROM jsonb_to_recordset($1::jsonb)
AS hashes(searched_hash bigint, found_hash bigint, distance bigint)
)
SELECT
'FurAffinity' site,
submission.id,
submission.hash_int hash,
submission.url,
submission.filename,
ARRAY(SELECT artist.name) artists,
submission.file_id,
null sources,
submission.rating,
submission.posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN submission ON hashes.found_hash = submission.hash_int
JOIN artist ON submission.artist_id = artist.id
WHERE hash_int IN (SELECT hashes.found_hash)
UNION ALL
SELECT
'e621' site,
e621.id,
e621.hash,
e621.data->'file'->>'url' url,
(e621.data->'file'->>'md5') || '.' || (e621.data->'file'->>'ext') filename,
ARRAY(SELECT jsonb_array_elements_text(e621.data->'tags'->'artist')) artists,
null file_id,
ARRAY(SELECT jsonb_array_elements_text(e621.data->'sources')) sources,
e621.data->>'rating' rating,
to_timestamp(data->>'created_at', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN e621 ON hashes.found_hash = e621.hash
WHERE e621.hash IN (SELECT hashes.found_hash)
UNION ALL
SELECT
'Weasyl' site,
weasyl.id,
weasyl.hash,
weasyl.data->>'link' url,
null filename,
ARRAY(SELECT weasyl.data->>'owner_login') artists,
null file_id,
null sources,
weasyl.data->>'rating' rating,
to_timestamp(data->>'posted_at', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN weasyl ON hashes.found_hash = weasyl.hash
WHERE weasyl.hash IN (SELECT hashes.found_hash)
UNION ALL
SELECT
'Twitter' site,
tweet.id,
tweet_media.hash,
tweet_media.url,
null filename,
ARRAY(SELECT tweet.data->'user'->>'screen_name') artists,
null file_id,
null sources,
CASE
WHEN (tweet.data->'possibly_sensitive')::boolean IS true THEN 'adult'
WHEN (tweet.data->'possibly_sensitive')::boolean IS false THEN 'general'
END rating,
to_timestamp(tweet.data->>'created_at', 'DY Mon DD HH24:MI:SS +0000 YYYY') posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN tweet_media ON hashes.found_hash = tweet_media.hash
JOIN tweet ON tweet_media.tweet_id = tweet.id
WHERE tweet_media.hash IN (SELECT hashes.found_hash)

7
fuzzysearch-api/queries/update_rate_limit.sql Normal file
View File

@ -0,0 +1,7 @@
INSERT INTO
rate_limit (api_key_id, time_window, group_name, count)
VALUES
($1, $2, $3, $4) ON CONFLICT ON CONSTRAINT unique_window DO
UPDATE
set
count = rate_limit.count + $4 RETURNING rate_limit.count

130
fuzzysearch-api/src/filters.rs
View File

@ -1,130 +0,0 @@
use crate::{handlers, Pool};
use crate::{types::*, Endpoints};
use std::convert::Infallible;
use tracing_futures::Instrument;
use warp::{Filter, Rejection, Reply};
pub fn search(
db: Pool,
bkapi: bkapi_client::BKApiClient,
endpoints: Endpoints,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
search_image(db.clone(), bkapi.clone(), endpoints)
.or(search_hashes(db.clone(), bkapi.clone()))
.or(search_file(db.clone()))
.or(check_handle(db.clone()))
.or(search_image_by_url(db, bkapi))
}
pub fn search_file(db: Pool) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path("file")
.and(warp::header::headers_cloned())
.and(warp::get())
.and(warp::query::<FileSearchOpts>())
.and(with_pool(db))
.and(with_api_key())
.and_then(|headers, opts, db, api_key| {
use tracing_opentelemetry::OpenTelemetrySpanExt;
let span = tracing::info_span!("search_file", ?opts);
span.set_parent(with_telem(headers));
span.in_scope(|| handlers::search_file(opts, db, api_key).in_current_span())
})
}
pub fn search_image(
db: Pool,
bkapi: bkapi_client::BKApiClient,
endpoints: Endpoints,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path("image")
.and(warp::header::headers_cloned())
.and(warp::post())
.and(warp::multipart::form().max_length(1024 * 1024 * 10))
.and(warp::query::<ImageSearchOpts>())
.and(with_pool(db))
.and(with_bkapi(bkapi))
.and(with_api_key())
.and(with_endpoints(endpoints))
.and_then(|headers, form, opts, pool, bkapi, api_key, endpoints| {
use tracing_opentelemetry::OpenTelemetrySpanExt;
let span = tracing::info_span!("search_image", ?opts);
span.set_parent(with_telem(headers));
span.in_scope(|| {
handlers::search_image(form, opts, pool, bkapi, api_key, endpoints)
.in_current_span()
})
})
}
pub fn search_image_by_url(
db: Pool,
bkapi: bkapi_client::BKApiClient,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path("url")
.and(warp::get())
.and(warp::query::<UrlSearchOpts>())
.and(with_pool(db))
.and(with_bkapi(bkapi))
.and(with_api_key())
.and_then(handlers::search_image_by_url)
}
pub fn search_hashes(
db: Pool,
bkapi: bkapi_client::BKApiClient,
) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path("hashes")
.and(warp::header::headers_cloned())
.and(warp::get())
.and(warp::query::<HashSearchOpts>())
.and(with_pool(db))
.and(with_bkapi(bkapi))
.and(with_api_key())
.and_then(|headers, opts, db, bkapi, api_key| {
use tracing_opentelemetry::OpenTelemetrySpanExt;
let span = tracing::info_span!("search_hashes", ?opts);
span.set_parent(with_telem(headers));
span.in_scope(|| handlers::search_hashes(opts, db, bkapi, api_key).in_current_span())
})
}
pub fn check_handle(db: Pool) -> impl Filter<Extract = impl Reply, Error = Rejection> + Clone {
warp::path("handle")
.and(warp::get())
.and(warp::query::<HandleOpts>())
.and(with_pool(db))
.and_then(handlers::check_handle)
}
fn with_api_key() -> impl Filter<Extract = (String,), Error = Rejection> + Clone {
warp::header::<String>("x-api-key")
}
fn with_pool(db: Pool) -> impl Filter<Extract = (Pool,), Error = Infallible> + Clone {
warp::any().map(move || db.clone())
}
fn with_bkapi(
bkapi: bkapi_client::BKApiClient,
) -> impl Filter<Extract = (bkapi_client::BKApiClient,), Error = Infallible> + Clone {
warp::any().map(move || bkapi.clone())
}
fn with_endpoints(
endpoints: Endpoints,
) -> impl Filter<Extract = (Endpoints,), Error = Infallible> + Clone {
warp::any().map(move || endpoints.clone())
}
fn with_telem(headers: warp::http::HeaderMap) -> opentelemetry::Context {
let remote_context = opentelemetry::global::get_text_map_propagator(|propagator| {
propagator.extract(&opentelemetry_http::HeaderExtractor(&headers))
});
tracing::trace!(?remote_context, "Got remote context");
remote_context
}

515
fuzzysearch-api/src/handlers.rs
View File

@ -1,515 +0,0 @@
use futures::StreamExt;
use futures::TryStreamExt;
use hyper::StatusCode;
use lazy_static::lazy_static;
use prometheus::{register_histogram, register_int_counter, Histogram, IntCounter};
use std::convert::TryInto;
use tracing::{span, warn};
use tracing_futures::Instrument;
use warp::{Rejection, Reply};
use crate::models::image_query;
use crate::types::*;
use crate::Endpoints;
use crate::{early_return, rate_limit, Pool};
use fuzzysearch_common::{
trace::InjectContext,
types::{SearchResult, SiteInfo},
};
lazy_static! {
static ref IMAGE_HASH_DURATION: Histogram = register_histogram!(
"fuzzysearch_api_image_hash_seconds",
"Duration to perform an image hash operation"
)
.unwrap();
static ref VIDEO_HASH_DURATION: Histogram = register_histogram!(
"fuzzysearch_api_video_hash_seconds",
"Duration to perform a video hash operation"
)
.unwrap();
static ref IMAGE_URL_DOWNLOAD_DURATION: Histogram = register_histogram!(
"fuzzysearch_api_image_url_download_seconds",
"Duration to download an image from a provided URL"
)
.unwrap();
static ref UNHANDLED_REJECTIONS: IntCounter = register_int_counter!(
"fuzzysearch_api_unhandled_rejections_count",
"Number of unhandled HTTP rejections"
)
.unwrap();
}
#[derive(Debug)]
enum Error {
Postgres(sqlx::Error),
Reqwest(reqwest::Error),
Warp(warp::Error),
InvalidData,
InvalidImage,
ApiKey,
RateLimit,
}
impl warp::Reply for Error {
fn into_response(self) -> warp::reply::Response {
let msg = match self {
Error::Postgres(_) | Error::Reqwest(_) | Error::Warp(_) => ErrorMessage {
code: 500,
message: "Internal server error".to_string(),
},
Error::InvalidData => ErrorMessage {
code: 400,
message: "Invalid data provided".to_string(),
},
Error::InvalidImage => ErrorMessage {
code: 400,
message: "Invalid image provided".to_string(),
},
Error::ApiKey => ErrorMessage {
code: 401,
message: "Invalid API key".to_string(),
},
Error::RateLimit => ErrorMessage {
code: 429,
message: "Too many requests".to_string(),
},
};
let body = hyper::body::Body::from(serde_json::to_string(&msg).unwrap());
warp::http::Response::builder()
.status(msg.code)
.body(body)
.unwrap()
}
}
impl From<sqlx::Error> for Error {
fn from(err: sqlx::Error) -> Self {
Error::Postgres(err)
}
}
impl From<reqwest::Error> for Error {
fn from(err: reqwest::Error) -> Self {
Error::Reqwest(err)
}
}
impl From<warp::Error> for Error {
fn from(err: warp::Error) -> Self {
Self::Warp(err)
}
}
#[tracing::instrument(skip(endpoints, form))]
async fn hash_input(
endpoints: &Endpoints,
mut form: warp::multipart::FormData,
) -> Result<i64, Error> {
let mut image_part = None;
tracing::debug!("looking at image parts");
while let Ok(Some(part)) = form.try_next().await {
if part.name() == "image" {
image_part = Some(part);
}
}
let image_part = image_part.ok_or(Error::InvalidImage)?;
tracing::debug!("found image part, reading data");
let bytes = image_part
.stream()
.fold(bytes::BytesMut::new(), |mut buf, chunk| {
use bytes::BufMut;
buf.put(chunk.unwrap());
async move { buf }
})
.await;
let part = reqwest::multipart::Part::bytes(bytes.to_vec());
let form = reqwest::multipart::Form::new().part("image", part);
tracing::debug!("sending image to hash input service");
let client = reqwest::Client::new();
let resp = client
.post(&endpoints.hash_input)
.inject_context()
.multipart(form)
.send()
.await?;
tracing::debug!("got response");
if resp.status() != StatusCode::OK {
return Err(Error::InvalidImage);
}
let hash: i64 = resp
.text()
.await?
.parse()
.map_err(|_err| Error::InvalidImage)?;
Ok(hash)
}
pub async fn search_image(
form: warp::multipart::FormData,
opts: ImageSearchOpts,
db: Pool,
bkapi: bkapi_client::BKApiClient,
api_key: String,
endpoints: Endpoints,
) -> Result<Box<dyn Reply>, Rejection> {
let image_remaining = rate_limit!(&api_key, &db, image_limit, "image");
let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash");
let num = early_return!(hash_input(&endpoints, form).await);
let mut items = {
if opts.search_type == Some(ImageSearchType::Force) {
image_query(db.clone(), bkapi.clone(), vec![num], 10)
.await
.unwrap()
} else {
let results = image_query(db.clone(), bkapi.clone(), vec![num], 0)
.await
.unwrap();
if results.is_empty() && opts.search_type != Some(ImageSearchType::Exact) {
image_query(db.clone(), bkapi.clone(), vec![num], 10)
.await
.unwrap()
} else {
results
}
}
};
items.sort_by(|a, b| {
a.distance
.unwrap_or(u64::max_value())
.partial_cmp(&b.distance.unwrap_or(u64::max_value()))
.unwrap()
});
let similarity = ImageSimilarity {
hash: num,
matches: items,
};
let resp = warp::http::Response::builder()
.header("x-image-hash", num.to_string())
.header("x-rate-limit-total-image", image_remaining.1.to_string())
.header(
"x-rate-limit-remaining-image",
image_remaining.0.to_string(),
)
.header("x-rate-limit-total-hash", hash_remaining.1.to_string())
.header("x-rate-limit-remaining-hash", hash_remaining.0.to_string())
.header("content-type", "application/json")
.body(serde_json::to_string(&similarity).unwrap())
.unwrap();
Ok(Box::new(resp))
}
pub async fn search_hashes(
opts: HashSearchOpts,
db: Pool,
bkapi: bkapi_client::BKApiClient,
api_key: String,
) -> Result<Box<dyn Reply>, Rejection> {
let pool = db.clone();
let hashes: Vec<i64> = opts
.hashes
.split(',')
.take(10)
.filter_map(|hash| hash.parse::<i64>().ok())
.collect();
if hashes.is_empty() {
return Ok(Box::new(Error::InvalidData));
}
let image_remaining = rate_limit!(&api_key, &db, image_limit, "image", hashes.len() as i16);
let results =
early_return!(image_query(pool, bkapi, hashes.clone(), opts.distance.unwrap_or(10)).await);
let resp = warp::http::Response::builder()
.header("x-rate-limit-total-image", image_remaining.1.to_string())
.header(
"x-rate-limit-remaining-image",
image_remaining.0.to_string(),
)
.header("content-type", "application/json")
.body(serde_json::to_string(&results).unwrap())
.unwrap();
Ok(Box::new(resp))
}
pub async fn search_file(
opts: FileSearchOpts,
db: Pool,
api_key: String,
) -> Result<Box<dyn Reply>, Rejection> {
use sqlx::Row;
let file_remaining = rate_limit!(&api_key, &db, name_limit, "file");
let query = if let Some(ref id) = opts.id {
sqlx::query(
"SELECT
submission.id,
submission.url,
submission.filename,
submission.file_id,
submission.rating,
submission.posted_at,
artist.name,
hashes.id hash_id
FROM
submission
JOIN artist
ON artist.id = submission.artist_id
JOIN hashes
ON hashes.furaffinity_id = submission.id
WHERE
file_id = $1
LIMIT 10",
)
.bind(id)
} else if let Some(ref name) = opts.name {
sqlx::query(
"SELECT
submission.id,
submission.url,
submission.filename,
submission.file_id,
submission.rating,
submission.posted_at,
artist.name,
hashes.id hash_id
FROM
submission
JOIN artist
ON artist.id = submission.artist_id
JOIN hashes
ON hashes.furaffinity_id = submission.id
WHERE
lower(filename) = lower($1)
LIMIT 10",
)
.bind(name)
} else if let Some(ref url) = opts.url {
sqlx::query(
"SELECT
submission.id,
submission.url,
submission.filename,
submission.file_id,
submission.rating,
submission.posted_at,
artist.name,
hashes.id hash_id
FROM
submission
JOIN artist
ON artist.id = submission.artist_id
JOIN hashes
ON hashes.furaffinity_id = submission.id
WHERE
lower(url) = lower($1)
LIMIT 10",
)
.bind(url)
} else if let Some(ref site_id) = opts.site_id {
sqlx::query(
"SELECT
submission.id,
submission.url,
submission.filename,
submission.file_id,
submission.rating,
submission.posted_at,
artist.name,
hashes.id hash_id
FROM
submission
JOIN artist
ON artist.id = submission.artist_id
JOIN hashes
ON hashes.furaffinity_id = submission.id
WHERE
submission.id = $1
LIMIT 10",
)
.bind(site_id)
} else {
return Ok(Box::new(Error::InvalidData));
};
let matches: Result<Vec<SearchResult>, _> = query
.map(|row| SearchResult {
site_id: row.get::<i32, _>("id") as i64,
site_id_str: row.get::<i32, _>("id").to_string(),
url: row.get("url"),
filename: row.get("filename"),
posted_at: row.get("posted_at"),
artists: row
.get::<Option<String>, _>("name")
.map(|artist| vec![artist]),
distance: None,
hash: None,
searched_hash: None,
site_info: Some(SiteInfo::FurAffinity {
file_id: row.get("file_id"),
}),
rating: row
.get::<Option<String>, _>("rating")
.and_then(|rating| rating.parse().ok()),
})
.fetch_all(&db)
.await;
let matches = early_return!(matches);
let resp = warp::http::Response::builder()
.header("x-rate-limit-total-file", file_remaining.1.to_string())
.header("x-rate-limit-remaining-file", file_remaining.0.to_string())
.header("content-type", "application/json")
.body(serde_json::to_string(&matches).unwrap())
.unwrap();
Ok(Box::new(resp))
}
pub async fn check_handle(opts: HandleOpts, db: Pool) -> Result<Box<dyn Reply>, Rejection> {
let exists = if let Some(handle) = opts.twitter {
let result = sqlx::query_scalar!("SELECT exists(SELECT 1 FROM twitter_user WHERE lower(data->>'screen_name') = lower($1))", handle)
.fetch_optional(&db)
.await
.map(|row| row.flatten().unwrap_or(false));
early_return!(result)
} else {
false
};
Ok(Box::new(warp::reply::json(&exists)))
}
pub async fn search_image_by_url(
opts: UrlSearchOpts,
db: Pool,
bkapi: bkapi_client::BKApiClient,
api_key: String,
) -> Result<Box<dyn Reply>, Rejection> {
use bytes::BufMut;
let url = opts.url;
let image_remaining = rate_limit!(&api_key, &db, image_limit, "image");
let hash_remaining = rate_limit!(&api_key, &db, hash_limit, "hash");
let _timer = IMAGE_URL_DOWNLOAD_DURATION.start_timer();
let mut resp = match reqwest::get(&url).await {
Ok(resp) => resp,
Err(_err) => return Ok(Box::new(Error::InvalidImage)),
};
let content_length = resp
.headers()
.get("content-length")
.and_then(|len| {
String::from_utf8_lossy(len.as_bytes())
.parse::<usize>()
.ok()
})
.unwrap_or(0);
if content_length > 10_000_000 {
return Ok(Box::new(Error::InvalidImage));
}
let mut buf = bytes::BytesMut::with_capacity(content_length);
while let Some(chunk) = early_return!(resp.chunk().await) {
if buf.len() + chunk.len() > 10_000_000 {
return Ok(Box::new(Error::InvalidImage));
}
buf.put(chunk);
}
drop(_timer);
let _timer = IMAGE_HASH_DURATION.start_timer();
let hash = tokio::task::spawn_blocking(move || {
let hasher = fuzzysearch_common::get_hasher();
let image = image::load_from_memory(&buf).unwrap();
hasher.hash_image(&image)
})
.instrument(span!(tracing::Level::TRACE, "hashing image"))
.await
.unwrap();
drop(_timer);
let hash: [u8; 8] = hash.as_bytes().try_into().unwrap();
let num = i64::from_be_bytes(hash);
let results = image_query(db.clone(), bkapi.clone(), vec![num], 3)
.await
.unwrap();
let resp = warp::http::Response::builder()
.header("x-image-hash", num.to_string())
.header("x-rate-limit-total-image", image_remaining.1.to_string())
.header(
"x-rate-limit-remaining-image",
image_remaining.0.to_string(),
)
.header("x-rate-limit-total-hash", hash_remaining.1.to_string())
.header("x-rate-limit-remaining-hash", hash_remaining.0.to_string())
.header("content-type", "application/json")
.body(serde_json::to_string(&results).unwrap())
.unwrap();
Ok(Box::new(resp))
}
#[tracing::instrument]
pub async fn handle_rejection(err: Rejection) -> Result<Box<dyn Reply>, std::convert::Infallible> {
warn!("had rejection");
UNHANDLED_REJECTIONS.inc();
let (code, message) = if err.is_not_found() {
(
warp::http::StatusCode::NOT_FOUND,
"This page does not exist",
)
} else if err.find::<warp::reject::InvalidQuery>().is_some() {
return Ok(Box::new(Error::InvalidData) as Box<dyn Reply>);
} else if err.find::<warp::reject::MethodNotAllowed>().is_some() {
return Ok(Box::new(Error::InvalidData) as Box<dyn Reply>);
} else {
(
warp::http::StatusCode::INTERNAL_SERVER_ERROR,
"An unknown error occured",
)
};
let json = warp::reply::json(&ErrorMessage {
code: code.as_u16(),
message: message.into(),
});
Ok(Box::new(warp::reply::with_status(json, code)))
}

612
fuzzysearch-api/src/main.rs
View File

@ -1,12 +1,16 @@
#![recursion_limit = "256"] use std::{borrow::Cow, str::FromStr};
use warp::Filter; use bkapi_client::BKApiClient;
use bytes::BufMut;
mod filters; use hyper::StatusCode;
mod handlers; use poem::{error::ResponseError, listener::TcpListener, web::Data, EndpointExt, Request, Route};
mod models; use poem_openapi::{
mod types; auth::ApiKey,
mod utils; param::{Path, Query},
payload::{Json, Response},
types::multipart::Upload,
Multipart, Object, OneOf, OpenApi, OpenApiService, SecurityScheme,
};
type Pool = sqlx::PgPool; type Pool = sqlx::PgPool;
@ -16,14 +20,566 @@ pub struct Endpoints {
pub bkapi: String, pub bkapi: String,
} }
struct Api;
/// Simple authentication using a static API key. Must be manually requested.
#[derive(SecurityScheme)]
#[oai(
type = "api_key",
key_name = "X-Api-Key",
in = "header",
checker = "api_checker"
)]
struct ApiKeyAuthorization(UserApiKey);
struct UserApiKey {
id: i32,
name: Option<String>,
owner_email: String,
name_limit: i16,
image_limit: i16,
hash_limit: i16,
}
async fn api_checker(req: &Request, api_key: ApiKey) -> Option<UserApiKey> {
let pool: &Pool = req.data().unwrap();
sqlx::query_file_as!(UserApiKey, "queries/lookup_api_key.sql", api_key.key)
.fetch_optional(pool)
.await
.ok()
.flatten()
}
#[derive(poem_openapi::Enum, Debug, PartialEq)]
#[oai(rename_all = "snake_case")]
enum KnownServiceName {
Twitter,
}
#[derive(poem_openapi::Enum, Debug)]
#[oai(rename_all = "lowercase")]
enum Rating {
General,
Mature,
Adult,
}
impl FromStr for Rating {
type Err = String;
fn from_str(s: &str) -> Result<Self, Self::Err> {
let rating = match s {
"g" | "s" | "general" => Self::General,
"m" | "q" | "mature" => Self::Mature,
"a" | "e" | "adult" | "explicit" => Self::Adult,
_ => return Err(format!("unknown rating: {}", s)),
};
Ok(rating)
}
}
#[derive(Object, Debug)]
#[oai(rename = "FurAffinity")]
struct FurAffinityExtra {
file_id: i32,
}
#[derive(Object, Debug)]
#[oai(rename = "e621")]
struct E621Extra {
sources: Vec<String>,
}
#[derive(OneOf, Debug)]
#[oai(property_name = "site")]
enum SiteExtraData {
FurAffinity(FurAffinityExtra),
E621(E621Extra),
}
#[derive(Object, Debug)]
struct HashLookupResult {
site_name: String,
site_id: i64,
site_id_str: String,
site_extra_data: Option<SiteExtraData>,
url: String,
filename: String,
artists: Option<Vec<String>>,
rating: Option<Rating>,
posted_at: Option<chrono::DateTime<chrono::Utc>>,
hash: i64,
searched_hash: i64,
distance: u64,
}
#[derive(serde::Serialize)]
struct HashSearch {
searched_hash: i64,
found_hash: i64,
distance: u64,
}
#[derive(Debug, thiserror::Error)]
enum Error {
#[error("database error: {0}")]
Database(#[from] sqlx::Error),
#[error("network error: {0}")]
Network(#[from] reqwest::Error),
#[error("json error: {0}")]
Json(#[from] serde_json::Error),
#[error("bad request: {0}")]
BadRequest(#[from] BadRequest),
}
impl ResponseError for Error {
fn status(&self) -> hyper::StatusCode {
hyper::StatusCode::INTERNAL_SERVER_ERROR
}
}
#[derive(Debug, thiserror::Error)]
#[error("bad request: {message}")]
struct BadRequest {
message: Cow<'static, str>,
}
impl BadRequest {
fn with_message<M: Into<Cow<'static, str>>>(message: M) -> Self {
Self {
message: message.into(),
}
}
}
impl ResponseError for BadRequest {
fn status(&self) -> hyper::StatusCode {
hyper::StatusCode::BAD_REQUEST
}
}
#[derive(Debug, thiserror::Error)]
#[error("rate limited")]
struct RateLimited {
bucket: String,
}
impl ResponseError for RateLimited {
fn status(&self) -> hyper::StatusCode {
hyper::StatusCode::TOO_MANY_REQUESTS
}
}
/// The status of an API key's rate limit.
#[derive(Debug, PartialEq)]
pub enum RateLimit {
/// This key is limited, we should deny the request.
Limited,
/// This key is available, contains the number of requests made.
Available((i16, i16)),
}
async fn update_rate_limit(
pool: &Pool,
key_id: i32,
key_group_limit: i16,
group_name: &'static str,
incr_by: i16,
) -> Result<RateLimit, sqlx::Error> {
let now = chrono::Utc::now();
let timestamp = now.timestamp();
let time_window = timestamp - (timestamp % 60);
let count: i16 = sqlx::query_file_scalar!(
"queries/update_rate_limit.sql",
key_id,
time_window,
group_name,
incr_by
)
.fetch_one(pool)
.await?;
if count > key_group_limit {
Ok(RateLimit::Limited)
} else {
Ok(RateLimit::Available((
key_group_limit - count,
key_group_limit,
)))
}
}
macro_rules! rate_limit {
($api_key:expr, $db:expr, $limit:tt, $group:expr) => {
rate_limit!($api_key, $db, $limit, $group, 1)
};
($api_key:expr, $db:expr, $limit:tt, $group:expr, $incr_by:expr) => {{
let rate_limit = update_rate_limit($db, $api_key.0.id, $api_key.0.$limit, $group, $incr_by)
.await
.map_err(Error::from)?;
match rate_limit {
RateLimit::Limited => {
return Err(RateLimited {
bucket: $group.to_string(),
}
.into())
}
RateLimit::Available(count) => count,
}
}};
}
async fn lookup_hashes(
pool: &Pool,
bkapi: &BKApiClient,
hashes: &[i64],
distance: u64,
) -> Result<Vec<HashLookupResult>, Error> {
if distance > 10 {
return Err(BadRequest::with_message(format!("distance too large: {}", distance)).into());
}
let index_hashes: Vec<_> = bkapi
.search_many(hashes, distance)
.await?
.into_iter()
.flat_map(|results| {
let hash = results.hash;
results.hashes.into_iter().map(move |result| HashSearch {
searched_hash: hash,
found_hash: result.hash,
distance: result.distance,
})
})
.collect();
let data = serde_json::to_value(index_hashes)?;
let results = sqlx::query_file!("queries/lookup_hashes.sql", data)
.map(|row| {
let site_extra_data = match row.site.as_deref() {
Some("FurAffinity") => Some(SiteExtraData::FurAffinity(FurAffinityExtra {
file_id: row.file_id.unwrap_or(-1),
})),
Some("e621") => Some(SiteExtraData::E621(E621Extra {
sources: row.sources.unwrap_or_default(),
})),
_ => None,
};
HashLookupResult {
site_name: row.site.unwrap_or_default(),
site_id: row.id.unwrap_or_default(),
site_id_str: row.id.unwrap_or_default().to_string(),
site_extra_data,
url: row.url.unwrap_or_default(),
filename: row.filename.unwrap_or_default(),
artists: row.artists,
posted_at: row.posted_at,
rating: row.rating.and_then(|rating| rating.parse().ok()),
hash: row.hash.unwrap_or_default(),
searched_hash: row.searched_hash.unwrap_or_default(),
distance: row.distance.unwrap_or_default() as u64,
}
})
.fetch_all(pool)
.await?;
Ok(results)
}
#[derive(Debug, Multipart)]
struct ImageSearchPayload {
image: Upload,
}
async fn hash_input(
client: &reqwest::Client,
hash_input_endpoint: &str,
image: reqwest::Body,
) -> Result<i64, Error> {
let part = reqwest::multipart::Part::stream(image);
let form = reqwest::multipart::Form::new().part("image", part);
let resp = client
.post(hash_input_endpoint)
.multipart(form)
.send()
.await?;
if resp.status() != StatusCode::OK {
return Err(BadRequest::with_message("invalid image").into());
}
match resp.text().await?.parse() {
Ok(hash) => Ok(hash),
Err(_err) => Err(BadRequest::with_message("invalid image").into()),
}
}
#[derive(poem_openapi::Enum, Debug, PartialEq)]
#[oai(rename_all = "lowercase")]
enum ImageSearchType {
Force,
Close,
Exact,
}
#[derive(Object, Debug)]
struct ImageSearchResult {
hash: i64,
matches: Vec<HashLookupResult>,
}
#[derive(Object, Debug)]
struct FurAffinityFile {
id: i32,
url: Option<String>,
filename: Option<String>,
file_id: Option<i32>,
rating: Option<Rating>,
posted_at: Option<chrono::DateTime<chrono::Utc>>,
artist: Option<String>,
hash: Option<i64>,
}
#[OpenApi]
impl Api {
/// Lookup images by hash
///
/// Perform a lookup for up to 10 given hashes.
#[oai(path = "/hashes", method = "get")]
async fn hashes(
&self,
pool: Data<&Pool>,
bkapi: Data<&BKApiClient>,
auth: ApiKeyAuthorization,
hashes: Query<String>,
distance: Query<Option<u64>>,
) -> poem::Result<Response<Json<Vec<HashLookupResult>>>> {
let hashes: Vec<i64> = hashes
.0
.split(',')
.take(10)
.filter_map(|hash| hash.parse().ok())
.collect();
let image_remaining = rate_limit!(auth, pool.0, image_limit, "image", hashes.len() as i16);
if hashes.is_empty() {
return Err(BadRequest::with_message("hashes must be provided").into());
}
let results = lookup_hashes(&pool, &bkapi, &hashes, distance.unwrap_or(3)).await?;
let resp = Response::new(Json(results))
.header("x-rate-limit-total-image", image_remaining.1)
.header("x-rate-limit-remaining-image", image_remaining.0);
Ok(resp)
}
/// Lookup images by image
///
/// Perform a lookup with a given image.
#[oai(path = "/image", method = "post")]
async fn image(
&self,
pool: Data<&Pool>,
bkapi: Data<&BKApiClient>,
client: Data<&reqwest::Client>,
endpoints: Data<&Endpoints>,
auth: ApiKeyAuthorization,
search_type: Query<Option<ImageSearchType>>,
payload: ImageSearchPayload,
) -> poem::Result<Response<Json<ImageSearchResult>>> {
let image_remaining = rate_limit!(auth, pool.0, image_limit, "image");
let hash_remaining = rate_limit!(auth, pool.0, hash_limit, "hash");
let stream = tokio_util::io::ReaderStream::new(payload.image.into_async_read());
let body = reqwest::Body::wrap_stream(stream);
let hash = hash_input(&client, &endpoints.hash_input, body).await?;
let search_type = search_type.0.unwrap_or(ImageSearchType::Close);
let hashes = vec![hash];
let mut results = {
if search_type == ImageSearchType::Force {
lookup_hashes(pool.0, bkapi.0, &hashes, 10).await?
} else {
let results = lookup_hashes(pool.0, bkapi.0, &hashes, 0).await?;
if results.is_empty() && search_type != ImageSearchType::Exact {
lookup_hashes(pool.0, bkapi.0, &hashes, 10).await?
} else {
results
}
}
};
results.sort_by(|a, b| a.distance.partial_cmp(&b.distance).unwrap());
let resp = Response::new(Json(ImageSearchResult {
hash,
matches: results,
}))
.header("x-image-hash", hash)
.header("x-rate-limit-total-image", image_remaining.1)
.header("x-rate-limit-remaining-image", image_remaining.0)
.header("x-rate-limit-total-hash", hash_remaining.1)
.header("x-rate-limit-remaining-hash", hash_remaining.0);
Ok(resp)
}
/// Lookup images by image URL
///
/// Perform a lookup for an image at the given URL. Image may not exceed 10MB.
#[oai(path = "/url", method = "get")]
async fn url(
&self,
pool: Data<&Pool>,
bkapi: Data<&BKApiClient>,
client: Data<&reqwest::Client>,
endpoints: Data<&Endpoints>,
auth: ApiKeyAuthorization,
url: Query<String>,
distance: Query<Option<u64>>,
) -> poem::Result<Response<Json<ImageSearchResult>>> {
let image_remaining = rate_limit!(auth, pool.0, image_limit, "image");
let hash_remaining = rate_limit!(auth, pool.0, hash_limit, "hash");
let mut resp = client.get(&url.0).send().await.map_err(Error::from)?;
let distance = distance.unwrap_or(3);
let content_length = resp
.headers()
.get("content-length")
.and_then(|len| {
String::from_utf8_lossy(len.as_bytes())
.parse::<usize>()
.ok()
})
.unwrap_or(0);
if content_length > 10_000_000 {
return Err(BadRequest::with_message(format!(
"image too large: {} bytes",
content_length
))
.into());
}
let mut buf = bytes::BytesMut::with_capacity(content_length);
while let Some(chunk) = resp.chunk().await.map_err(Error::from)? {
if buf.len() + chunk.len() > 10_000_000 {
return Err(BadRequest::with_message(format!(
"image too large: {} bytes",
content_length
))
.into());
}
buf.put(chunk);
}
let body = reqwest::Body::from(buf.to_vec());
let hash = hash_input(&client, &endpoints.hash_input, body).await?;
let results = lookup_hashes(pool.0, bkapi.0, &[hash], distance).await?;
let resp = Response::new(Json(ImageSearchResult {
hash,
matches: results,
}))
.header("x-image-hash", hash)
.header("x-rate-limit-total-image", image_remaining.1)
.header("x-rate-limit-remaining-image", image_remaining.0)
.header("x-rate-limit-total-hash", hash_remaining.1)
.header("x-rate-limit-remaining-hash", hash_remaining.0);
Ok(resp)
}
/// Lookup FurAffinity submission by File ID
#[oai(path = "/furaffinity/file_id", method = "get")]
async fn furaffinity_data(
&self,
pool: Data<&Pool>,
auth: ApiKeyAuthorization,
file_id: Query<i32>,
) -> poem::Result<Response<Json<Vec<FurAffinityFile>>>> {
let file_remaining = rate_limit!(auth, pool.0, image_limit, "file");
let matches = sqlx::query_file!("queries/lookup_furaffinity_file_id.sql", file_id.0)
.map(|row| FurAffinityFile {
id: row.id,
url: row.url,
filename: row.filename,
file_id: row.file_id,
rating: row.rating.and_then(|rating| rating.parse().ok()),
posted_at: row.posted_at,
artist: Some(row.artist),
hash: row.hash,
})
.fetch_all(pool.0)
.await
.map_err(Error::from)?;
let resp = Response::new(Json(matches))
.header("x-rate-limit-total-file", file_remaining.1)
.header("x-rate-limit-remaining-file", file_remaining.0);
Ok(resp)
}
/// Check if a handle is known for a given service
///
/// If the handle is known, the associated media items should be available
/// in the search index.
#[oai(path = "/known/:service", method = "get")]
async fn known_service(
&self,
pool: Data<&Pool>,
service: Path<KnownServiceName>,
handle: Query<String>,
) -> poem::Result<Json<bool>> {
let handle_exists = match service.0 {
KnownServiceName::Twitter => {
sqlx::query_file_scalar!("queries/handle_twitter.sql", handle.0)
.fetch_one(pool.0)
.await
.map_err(poem::error::InternalServerError)?
}
};
Ok(Json(handle_exists))
}
}
#[tokio::main] #[tokio::main]
async fn main() { async fn main() {
fuzzysearch_common::trace::configure_tracing("fuzzysearch-api"); fuzzysearch_common::trace::configure_tracing("fuzzysearch-api");
fuzzysearch_common::trace::serve_metrics().await; fuzzysearch_common::trace::serve_metrics().await;
let s = std::env::var("DATABASE_URL").expect("Missing DATABASE_URL"); let server_endpoint =
std::env::var("SERVER_ENDPOINT").unwrap_or_else(|_err| "http://localhost:8080".to_string());
let db_pool = sqlx::PgPool::connect(&s) let database_url = std::env::var("DATABASE_URL").expect("Missing DATABASE_URL");
let pool = sqlx::PgPool::connect(&database_url)
.await .await
.expect("Unable to create Postgres pool"); .expect("Unable to create Postgres pool");
@ -32,23 +588,29 @@ async fn main() {
bkapi: std::env::var("ENDPOINT_BKAPI").expect("Missing ENDPOINT_BKAPI"), bkapi: std::env::var("ENDPOINT_BKAPI").expect("Missing ENDPOINT_BKAPI"),
}; };
let bkapi = bkapi_client::BKApiClient::new(&endpoints.bkapi); let bkapi = BKApiClient::new(&endpoints.bkapi);
let log = warp::log("fuzzysearch-api"); let cors = poem::middleware::Cors::new()
let cors = warp::cors() .allow_methods([poem::http::Method::GET, poem::http::Method::POST]);
.allow_any_origin()
.allow_headers(vec!["x-api-key"])
.allow_methods(vec!["GET", "POST"]);
let options = warp::options().map(|| "✓"); let api_service = OpenApiService::new(Api, "FuzzySearch", "1.0").server(server_endpoint);
let api_spec_endpoint = api_service.spec_endpoint();
let api = options.or(filters::search(db_pool, bkapi, endpoints)); let docs = api_service.swagger_ui();
let routes = api let app = Route::new()
.or(warp::path::end() .nest("/", api_service)
.map(|| warp::redirect(warp::http::Uri::from_static("https://fuzzysearch.net")))) .nest("/docs", docs)
.with(log) .at("/openapi.json", api_spec_endpoint)
.with(cors) .data(pool)
.recover(handlers::handle_rejection); .data(bkapi)
.data(endpoints)
.data(reqwest::Client::new())
.with(poem::middleware::Tracing)
.with(poem::middleware::OpenTelemetryMetrics::new())
.with(cors);
warp::serve(routes).run(([0, 0, 0, 0], 8080)).await; poem::Server::new(TcpListener::bind("0.0.0.0:8080"))
.run(app)
.await
.unwrap();
} }

191
fuzzysearch-api/src/models.rs
View File

@ -1,191 +0,0 @@
use lazy_static::lazy_static;
use prometheus::{register_histogram, Histogram};
use crate::types::*;
use crate::Pool;
use fuzzysearch_common::types::{SearchResult, SiteInfo};
lazy_static! {
static ref IMAGE_QUERY_DURATION: Histogram = register_histogram!(
"fuzzysearch_api_image_query_seconds",
"Duration to perform a single image lookup query"
)
.unwrap();
}
#[tracing::instrument(skip(db))]
pub async fn lookup_api_key(key: &str, db: &sqlx::PgPool) -> Option<ApiKey> {
sqlx::query_as!(
ApiKey,
"SELECT
api_key.id,
api_key.name_limit,
api_key.image_limit,
api_key.hash_limit,
api_key.name,
account.email owner_email
FROM
api_key
JOIN account
ON account.id = api_key.user_id
WHERE
api_key.key = $1
",
key
)
.fetch_optional(db)
.await
.ok()
.flatten()
}
#[derive(serde::Serialize)]
struct HashSearch {
searched_hash: i64,
found_hash: i64,
distance: u64,
}
#[tracing::instrument(skip(pool, bkapi))]
pub async fn image_query(
pool: Pool,
bkapi: bkapi_client::BKApiClient,
hashes: Vec<i64>,
distance: i64,
) -> Result<Vec<SearchResult>, sqlx::Error> {
let found_hashes: Vec<HashSearch> = bkapi
.search_many(&hashes, distance as u64)
.await
.unwrap()
.into_iter()
.flat_map(|results| {
results
.hashes
.iter()
.map(|hash| HashSearch {
searched_hash: results.hash,
found_hash: hash.hash,
distance: hash.distance,
})
.collect::<Vec<_>>()
})
.collect();
let timer = IMAGE_QUERY_DURATION.start_timer();
let matches = sqlx::query!(
r#"WITH hashes AS (
SELECT * FROM jsonb_to_recordset($1::jsonb)
AS hashes(searched_hash bigint, found_hash bigint, distance bigint)
)
SELECT
'FurAffinity' site,
submission.id,
submission.hash_int hash,
submission.url,
submission.filename,
ARRAY(SELECT artist.name) artists,
submission.file_id,
null sources,
submission.rating,
submission.posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN submission ON hashes.found_hash = submission.hash_int
JOIN artist ON submission.artist_id = artist.id
WHERE hash_int IN (SELECT hashes.found_hash)
UNION ALL
SELECT
'e621' site,
e621.id,
e621.hash,
e621.data->'file'->>'url' url,
(e621.data->'file'->>'md5') || '.' || (e621.data->'file'->>'ext') filename,
ARRAY(SELECT jsonb_array_elements_text(e621.data->'tags'->'artist')) artists,
null file_id,
ARRAY(SELECT jsonb_array_elements_text(e621.data->'sources')) sources,
e621.data->>'rating' rating,
to_timestamp(data->>'created_at', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN e621 ON hashes.found_hash = e621.hash
WHERE e621.hash IN (SELECT hashes.found_hash)
UNION ALL
SELECT
'Weasyl' site,
weasyl.id,
weasyl.hash,
weasyl.data->>'link' url,
null filename,
ARRAY(SELECT weasyl.data->>'owner_login') artists,
null file_id,
null sources,
weasyl.data->>'rating' rating,
to_timestamp(data->>'posted_at', 'YYYY-MM-DD"T"HH24:MI:SS"Z"') posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN weasyl ON hashes.found_hash = weasyl.hash
WHERE weasyl.hash IN (SELECT hashes.found_hash)
UNION ALL
SELECT
'Twitter' site,
tweet.id,
tweet_media.hash,
tweet_media.url,
null filename,
ARRAY(SELECT tweet.data->'user'->>'screen_name') artists,
null file_id,
null sources,
CASE
WHEN (tweet.data->'possibly_sensitive')::boolean IS true THEN 'adult'
WHEN (tweet.data->'possibly_sensitive')::boolean IS false THEN 'general'
END rating,
to_timestamp(tweet.data->>'created_at', 'DY Mon DD HH24:MI:SS +0000 YYYY') posted_at,
hashes.searched_hash,
hashes.distance
FROM hashes
JOIN tweet_media ON hashes.found_hash = tweet_media.hash
JOIN tweet ON tweet_media.tweet_id = tweet.id
WHERE tweet_media.hash IN (SELECT hashes.found_hash)"#,
serde_json::to_value(&found_hashes).unwrap()
)
.map(|row| {
use std::convert::TryFrom;
let site_info = match row.site.as_deref() {
Some("FurAffinity") => SiteInfo::FurAffinity {
file_id: row.file_id.unwrap_or(-1),
},
Some("e621") => SiteInfo::E621 {
sources: row.sources,
},
Some("Twitter") => SiteInfo::Twitter,
Some("Weasyl") => SiteInfo::Weasyl,
_ => panic!("Got unknown site"),
};
SearchResult {
site_id: row.id.unwrap_or_default(),
site_info: Some(site_info),
rating: row.rating.and_then(|rating| rating.parse().ok()),
site_id_str: row.id.unwrap_or_default().to_string(),
url: row.url.unwrap_or_default(),
posted_at: row.posted_at,
hash: row.hash,
distance: row
.distance
.map(|distance| u64::try_from(distance).ok())
.flatten(),
artists: row.artists,
filename: row.filename.unwrap_or_default(),
searched_hash: row.searched_hash,
}
})
.fetch_all(&pool)
.await?;
timer.stop_and_record();
Ok(matches)
}

76
fuzzysearch-api/src/types.rs
View File

@ -1,76 +0,0 @@
use serde::{Deserialize, Serialize};
use fuzzysearch_common::types::SearchResult;
/// An API key representation from the database.alloc
///
/// May contain information about the owner, always has rate limit information.
/// Limits are the number of requests allowed per minute.
#[derive(Debug)]
pub struct ApiKey {
pub id: i32,
pub name: Option<String>,
pub owner_email: String,
pub name_limit: i16,
pub image_limit: i16,
pub hash_limit: i16,
}
/// The status of an API key's rate limit.
#[derive(Debug, PartialEq)]
pub enum RateLimit {
/// This key is limited, we should deny the request.
Limited,
/// This key is available, contains the number of requests made.
Available((i16, i16)),
}
#[derive(Debug, Deserialize)]
pub struct FileSearchOpts {
pub id: Option<i32>,
pub name: Option<String>,
pub url: Option<String>,
pub site_id: Option<i32>,
}
#[derive(Debug, Deserialize)]
pub struct ImageSearchOpts {
#[serde(rename = "type")]
pub search_type: Option<ImageSearchType>,
}
#[derive(Debug, Deserialize, PartialEq)]
#[serde(rename_all = "lowercase")]
pub enum ImageSearchType {
Close,
Exact,
Force,
}
#[derive(Debug, Serialize)]
pub struct ImageSimilarity {
pub hash: i64,
pub matches: Vec<SearchResult>,
}
#[derive(Serialize)]
pub struct ErrorMessage {
pub code: u16,
pub message: String,
}
#[derive(Debug, Deserialize)]
pub struct HashSearchOpts {
pub hashes: String,
pub distance: Option<i64>,
}
#[derive(Debug, Deserialize)]
pub struct HandleOpts {
pub twitter: Option<String>,
}
#[derive(Debug, Deserialize)]
pub struct UrlSearchOpts {
pub url: String,
}

108
fuzzysearch-api/src/utils.rs
View File

@ -1,108 +0,0 @@
use crate::types::*;
use lazy_static::lazy_static;
use prometheus::{register_int_counter_vec, IntCounterVec};
lazy_static! {
pub static ref RATE_LIMIT_STATUS: IntCounterVec = register_int_counter_vec!(
"fuzzysearch_api_rate_limit_count",
"Number of allowed and rate limited requests",
&["status"]
)
.unwrap();
}
#[macro_export]
macro_rules! rate_limit {
($api_key:expr, $db:expr, $limit:tt, $group:expr) => {
rate_limit!($api_key, $db, $limit, $group, 1)
};
($api_key:expr, $db:expr, $limit:tt, $group:expr, $incr_by:expr) => {{
let api_key = match crate::models::lookup_api_key($api_key, $db).await {
Some(api_key) => api_key,
None => return Ok(Box::new(Error::ApiKey)),
};
let rate_limit = match crate::utils::update_rate_limit(
$db,
api_key.id,
api_key.$limit,
$group,
$incr_by,
)
.await
{
Ok(rate_limit) => rate_limit,
Err(err) => return Ok(Box::new(Error::Postgres(err))),
};
match rate_limit {
crate::types::RateLimit::Limited => {
crate::utils::RATE_LIMIT_STATUS
.with_label_values(&["limited"])
.inc();
return Ok(Box::new(Error::RateLimit));
}
crate::types::RateLimit::Available(count) => {
crate::utils::RATE_LIMIT_STATUS
.with_label_values(&["allowed"])
.inc();
count
}
}
}};
}
#[macro_export]
macro_rules! early_return {
($val:expr) => {
match $val {
Ok(val) => val,
Err(err) => return Ok(Box::new(Error::from(err))),
}
};
}
/// Increment the rate limit for a group.
///
/// We need to specify the ID of the API key to increment, the key's limit for
/// the specified group, the name of the group we're incrementing, and the
/// amount to increment for this request. This should remain as 1 except for
/// joined requests.
#[tracing::instrument(skip(db))]
pub async fn update_rate_limit(
db: &sqlx::PgPool,
key_id: i32,
key_group_limit: i16,
group_name: &'static str,
incr_by: i16,
) -> Result<RateLimit, sqlx::Error> {
let now = chrono::Utc::now();
let timestamp = now.timestamp();
let time_window = timestamp - (timestamp % 60);
let count: i16 = sqlx::query_scalar!(
"INSERT INTO
rate_limit (api_key_id, time_window, group_name, count)
VALUES
($1, $2, $3, $4)
ON CONFLICT ON CONSTRAINT unique_window
DO UPDATE set count = rate_limit.count + $4
RETURNING rate_limit.count",
key_id,
time_window,
group_name,
incr_by
)
.fetch_one(db)
.await?;
if count > key_group_limit {
Ok(RateLimit::Limited)
} else {
Ok(RateLimit::Available((
key_group_limit - count,
key_group_limit,
)))
}
}