diff --git a/.drone.yml b/.drone.yml index 4f13291..ec42eb3 100644 --- a/.drone.yml +++ b/.drone.yml @@ -15,18 +15,4 @@ steps: - cargo build - cargo test -- name: build-latest - image: plugins/docker - settings: - auto_tag: true - password: - from_secret: docker_password - registry: registry.huefox.com - repo: registry.huefox.com/fuzzysearch - username: - from_secret: docker_username - when: - branch: - - master - ... diff --git a/Cargo.lock b/Cargo.lock index a4a776d..a24708e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12,6 +12,15 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aho-corasick" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7404febffaa47dac81aa44dba71523c9d069b1bdc50a77db41195149e17f68e5" +dependencies = [ + "memchr", +] + [[package]] name = "ansi_term" version = "0.12.1" @@ -76,7 +85,7 @@ checksum = "374bba43fc924d90393ee7768e6f75d223a98307a488fe5bc34b66c3e96932a6" dependencies = [ "async-trait", "futures", - "tokio", + "tokio 0.2.23", ] [[package]] @@ -88,8 +97,8 @@ dependencies = [ "async-trait", "bb8", "futures", - "tokio", - "tokio-postgres", + "tokio 0.2.23", + "tokio-postgres 0.5.5", ] [[package]] @@ -164,6 +173,12 @@ dependencies = [ "safemem", ] +[[package]] +name = "bumpalo" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" + [[package]] name = "byte-tools" version = "0.3.1" @@ -188,6 +203,12 @@ version = "0.5.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" +[[package]] +name = "bytes" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0dcbc35f504eb6fc275a6d20e4ebcda18cf50d40ba6fabff8c711fa16cb3b16" + [[package]] name = "cc" version = "1.0.66" @@ -215,6 +236,14 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "cfscrape" +version = "0.1.0" +source = "git+https://github.com/Syfaro/cfscrape-rs.git#439cc06b190f01d420090a0cbcd19332fff8f05b" +dependencies = [ + "pyo3", +] + [[package]] name = "chrono" version = "0.4.19" @@ -254,12 +283,38 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" +[[package]] +name = "console_error_panic_hook" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211" +dependencies = [ + "cfg-if 0.1.10", + "wasm-bindgen", +] + [[package]] name = "const_fn" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c478836e029dcef17fb47c89023448c64f781a046e0300e257ad8225ae59afab" +[[package]] +name = "core-foundation" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a89e2ae426ea83155dccf10c0fa6b1463ef6d5fcb44cee0b224a408fa640a62" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation-sys" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b" + [[package]] name = "cpuid-bool" version = "0.1.2" @@ -331,6 +386,43 @@ dependencies = [ "subtle", ] +[[package]] +name = "cssparser" +version = "0.27.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a" +dependencies = [ + "cssparser-macros", + "dtoa-short", + "itoa", + "matches", + "phf", + "proc-macro2", + "quote", + "smallvec", + "syn", +] + +[[package]] +name = "cssparser-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "ctor" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fbaabec2c953050352311293be5c6aba8e141ba19d6811862b232d6fd020484" +dependencies = [ + "quote", + "syn", +] + [[package]] name = "deflate" version = "0.8.6" @@ -341,6 +433,17 @@ dependencies = [ "byteorder", ] +[[package]] +name = "derive_more" +version = "0.99.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cb0e6161ad61ed084a36ba71fbba9e3ac5aee3606fb607fe08da6acbcf3d8c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.8.1" @@ -365,12 +468,36 @@ version = "0.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" +[[package]] +name = "dtoa-short" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59020b8513b76630c49d918c33db9f4c91638e7d3404a28084083b87e33f76f2" +dependencies = [ + "dtoa", +] + +[[package]] +name = "ego-tree" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a68a4904193147e0a8dec3314640e6db742afd5f6e634f428a6af230d9b3591" + [[package]] name = "either" version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457" +[[package]] +name = "encoding_rs" +version = "0.8.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "801bbab217d7f79c0062f4f7205b5d4427c6d1a7bd7aafdd1475f7c59d62b283" +dependencies = [ + "cfg-if 1.0.0", +] + [[package]] name = "fake-simd" version = "0.1.2" @@ -408,12 +535,39 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "flate2" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129" +dependencies = [ + "cfg-if 1.0.0", + "crc32fast", + "libc", + "miniz_oxide 0.4.3", +] + [[package]] name = "fnv" version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.0.0" @@ -446,6 +600,33 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" +[[package]] +name = "furaffinity-rs" +version = "0.1.0" +source = "git+https://github.com/Syfaro/furaffinity-rs#9367bb49bb2b0adf41d53f1afe787b01579043b4" +dependencies = [ + "cfscrape", + "chrono", + "image", + "img_hash", + "lazy_static", + "regex", + "reqwest", + "scraper", + "sha2", + "tokio 0.2.23", +] + +[[package]] +name = "futf" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c9c1ce3fa9336301af935ab852c437817d14cd33690446569392e65170aac3b" +dependencies = [ + "mac", + "new_debug_unreachable", +] + [[package]] name = "futures" version = "0.3.8" @@ -549,7 +730,7 @@ dependencies = [ "bb8", "bb8-postgres", "bk-tree", - "bytes", + "bytes 0.5.6", "chrono", "ffmpeg-next", "futures", @@ -562,8 +743,8 @@ dependencies = [ "opentelemetry", "opentelemetry-jaeger", "serde", - "tokio", - "tokio-postgres", + "tokio 0.2.23", + "tokio-postgres 0.5.5", "tracing", "tracing-futures", "tracing-opentelemetry", @@ -584,6 +765,32 @@ dependencies = [ "tracing", ] +[[package]] +name = "fuzzysearch-ingest-furaffinity" +version = "0.1.0" +dependencies = [ + "chrono", + "furaffinity-rs", + "hyper", + "lazy_static", + "postgres 0.18.1", + "prometheus 0.11.0", + "r2d2", + "r2d2_postgres", + "reqwest", + "tokio 0.2.23", + "tokio-postgres 0.5.5", +] + +[[package]] +name = "fxhash" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c" +dependencies = [ + "byteorder", +] + [[package]] name = "generator" version = "0.6.23" @@ -616,6 +823,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "getopts" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +dependencies = [ + "unicode-width", +] + [[package]] name = "getrandom" version = "0.1.15" @@ -627,6 +843,17 @@ dependencies = [ "wasi 0.9.0+wasi-snapshot-preview1", ] +[[package]] +name = "ghost" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a5bcf1bbeab73aa4cf2fde60a846858dc036163c7c33bec309f8d17de785479" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "gif" version = "0.11.1" @@ -649,7 +876,7 @@ version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e4728fd124914ad25e99e3d15a9361a879f6620f63cb56bbb08f95abb97a535" dependencies = [ - "bytes", + "bytes 0.5.6", "fnv", "futures-core", "futures-sink", @@ -657,8 +884,8 @@ dependencies = [ "http", "indexmap", "slab", - "tokio", - "tokio-util", + "tokio 0.2.23", + "tokio-util 0.3.1", "tracing", "tracing-futures", ] @@ -683,7 +910,7 @@ checksum = "ed18eb2459bf1a09ad2d6b1547840c3e5e62882fa09b9a6a20b1de8e3228848f" dependencies = [ "base64 0.12.3", "bitflags", - "bytes", + "bytes 0.5.6", "headers-core", "http", "mime", @@ -709,6 +936,12 @@ dependencies = [ "libc", ] +[[package]] +name = "hex" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "644f9158b2f133fd50f5fb3242878846d9eb792e445c893805ff0e3824006e35" + [[package]] name = "hmac" version = "0.9.0" @@ -719,13 +952,27 @@ dependencies = [ "digest 0.9.0", ] +[[package]] +name = "html5ever" +version = "0.25.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aafcf38a1a36118242d29b92e1b08ef84e67e4a5ed06e0a80be20e6a32bfed6b" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "http" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d569972648b2c512421b5f2a405ad6ac9666547189d0c5477a3f200f3e02f9" dependencies = [ - "bytes", + "bytes 0.5.6", "fnv", "itoa", ] @@ -736,7 +983,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13d5ff830006f7646652e057693569bfe0d51760c0085a071769d142a205111b" dependencies = [ - "bytes", + "bytes 0.5.6", "http", ] @@ -758,7 +1005,7 @@ version = "0.13.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ad767baac13b44d4529fcf58ba2cd0995e36e7b435bc5b039de6f47e880dbf" dependencies = [ - "bytes", + "bytes 0.5.6", "futures-channel", "futures-core", "futures-util", @@ -770,12 +1017,25 @@ dependencies = [ "itoa", "pin-project 1.0.2", "socket2", - "tokio", + "tokio 0.2.23", "tower-service", "tracing", "want", ] +[[package]] +name = "hyper-tls" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d979acc56dcb5b8dddba3917601745e877576475aa046df3226eabdecef78eed" +dependencies = [ + "bytes 0.5.6", + "hyper", + "native-tls", + "tokio 0.2.23", + "tokio-tls", +] + [[package]] name = "idna" version = "0.2.0" @@ -829,6 +1089,29 @@ dependencies = [ "hashbrown", ] +[[package]] +name = "indoc" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47741a8bc60fb26eb8d6e0238bbb26d8575ff623fdc97b1a2c00c050b9684ed8" +dependencies = [ + "indoc-impl", + "proc-macro-hack", +] + +[[package]] +name = "indoc-impl" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce046d161f000fffde5f432a0d034d0341dc152643b2598ed5bfce44c4f3a8f0" +dependencies = [ + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", + "unindent", +] + [[package]] name = "infer" version = "0.3.2" @@ -841,7 +1124,7 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19a8a95243d5a0398cae618ec29477c6e3cb631152be5c19481f80bc71559754" dependencies = [ - "bytes", + "bytes 0.5.6", ] [[package]] @@ -859,6 +1142,28 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4f4ebd0bd29be0f11973e9b3e219005661042a019fd757798c36a47c87852625" +[[package]] +name = "inventory" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fedd49de24d8c263613701406611410687148ae8c37cd6452650b250f753a0dd" +dependencies = [ + "ctor", + "ghost", + "inventory-impl", +] + +[[package]] +name = "inventory-impl" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddead8880bc50f57fcd3b5869a7f6ff92570bb4e8f6870c22e2483272f2256da" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "iovec" version = "0.1.4" @@ -868,6 +1173,12 @@ dependencies = [ "libc", ] +[[package]] +name = "ipnet" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47be2f14c678be2fdcab04ab1171db51b2762ce6f0a8ee87c8dd4a04ed216135" + [[package]] name = "itoa" version = "0.4.6" @@ -884,6 +1195,15 @@ dependencies = [ "rayon", ] +[[package]] +name = "js-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf3d7383929f7c9c7c2d0fa596f325832df98c3704f2c60553080f7127a58175" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "kernel32-sys" version = "0.2.2" @@ -953,6 +1273,29 @@ dependencies = [ "serde_json", ] +[[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] +name = "markup5ever" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aae38d669396ca9b707bfc3db254bc382ddb94f57cc5c235f34623a669a01dab" +dependencies = [ + "log", + "phf", + "phf_codegen", + "serde", + "serde_derive", + "serde_json", + "string_cache", + "string_cache_codegen", + "tendril", +] + [[package]] name = "matchers" version = "0.0.1" @@ -1043,6 +1386,19 @@ dependencies = [ "winapi 0.2.8", ] +[[package]] +name = "mio" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f33bc887064ef1fd66020c9adfc45bb9f33d75a42096c81e7c56c65b75dd1a8b" +dependencies = [ + "libc", + "log", + "miow 0.3.6", + "ntapi", + "winapi 0.3.9", +] + [[package]] name = "mio-named-pipes" version = "0.1.7" @@ -1050,7 +1406,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0840c1c50fd55e521b247f949c241c9997709f23bd7f023b9762cd561e935656" dependencies = [ "log", - "mio", + "mio 0.6.23", "miow 0.3.6", "winapi 0.3.9", ] @@ -1063,7 +1419,7 @@ checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" dependencies = [ "iovec", "libc", - "mio", + "mio 0.6.23", ] [[package]] @@ -1106,6 +1462,24 @@ dependencies = [ "twoway", ] +[[package]] +name = "native-tls" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fcc7939b5edc4e4f86b1b4a04bb1498afaaf871b1a6691838ed06fcb48d3a3f" +dependencies = [ + "lazy_static", + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + [[package]] name = "net2" version = "0.2.36" @@ -1117,6 +1491,18 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + +[[package]] +name = "nodrop" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb" + [[package]] name = "nom" version = "5.1.2" @@ -1127,6 +1513,15 @@ dependencies = [ "version_check", ] +[[package]] +name = "ntapi" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f6bb902e437b6d86e03cce10a7e2af662292c5dfef23b65899ea3ac9354ad44" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "num-complex" version = "0.2.4" @@ -1206,6 +1601,39 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5" +[[package]] +name = "openssl" +version = "0.10.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d575eff3665419f9b83678ff2815858ad9d11567e082f5ac1814baba4e2bcb4" +dependencies = [ + "bitflags", + "cfg-if 0.1.10", + "foreign-types", + "lazy_static", + "libc", + "openssl-sys", +] + +[[package]] +name = "openssl-probe" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77af24da69f9d9341038eba93a073b1fdaaa1b788221b00a69bce9e762cb32de" + +[[package]] +name = "openssl-sys" +version = "0.9.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a842db4709b604f0fe5d1170ae3565899be2ad3d9cbc72dedc789ac0511f78de" +dependencies = [ + "autocfg 1.0.1", + "cc", + "libc", + "pkg-config", + "vcpkg", +] + [[package]] name = "opentelemetry" version = "0.6.0" @@ -1216,7 +1644,7 @@ dependencies = [ "lazy_static", "percent-encoding", "pin-project 0.4.27", - "prometheus", + "prometheus 0.7.0", "rand 0.7.3", ] @@ -1264,6 +1692,25 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "paste" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca20c77d80be666aef2b45486da86238fabe33e38306bd3118fe4af33fa880" +dependencies = [ + "paste-impl", + "proc-macro-hack", +] + +[[package]] +name = "paste-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d95a7db200b97ef370c8e6de0088252f7e0dfff7d047a28528e47456c0fc98b6" +dependencies = [ + "proc-macro-hack", +] + [[package]] name = "peeking_take_while" version = "0.1.2" @@ -1282,7 +1729,43 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12" dependencies = [ + "phf_macros", "phf_shared", + "proc-macro-hack", +] + +[[package]] +name = "phf_codegen" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815" +dependencies = [ + "phf_generator", + "phf_shared", +] + +[[package]] +name = "phf_generator" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526" +dependencies = [ + "phf_shared", + "rand 0.7.3", +] + +[[package]] +name = "phf_macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", ] [[package]] @@ -1370,6 +1853,34 @@ dependencies = [ "miniz_oxide 0.3.7", ] +[[package]] +name = "postgres" +version = "0.17.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14d864cf6c2eabf1323afe4145ff273aad1898e4f2a3bcb30347715df8624a07" +dependencies = [ + "bytes 0.5.6", + "fallible-iterator", + "futures", + "log", + "tokio 0.2.23", + "tokio-postgres 0.5.5", +] + +[[package]] +name = "postgres" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25d05de0900dc0254bbad5e4b36b0f750c23ff44fbfcca4c1b9071f7ec1c764" +dependencies = [ + "bytes 0.5.6", + "fallible-iterator", + "futures", + "log", + "tokio 0.3.5", + "tokio-postgres 0.6.0", +] + [[package]] name = "postgres-protocol" version = "0.5.3" @@ -1378,7 +1889,7 @@ checksum = "4888a0e36637ab38d76cace88c1476937d617ad015f07f6b669cec11beacc019" dependencies = [ "base64 0.13.0", "byteorder", - "bytes", + "bytes 0.5.6", "fallible-iterator", "hmac", "md5", @@ -1394,7 +1905,8 @@ version = "0.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cfc08a7d94a80665de4a83942fa8db2fdeaf2f123fc0535e384dc4fff251efae" dependencies = [ - "bytes", + "bytes 0.5.6", + "chrono", "fallible-iterator", "postgres-protocol", ] @@ -1405,6 +1917,12 @@ version = "0.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac74c624d6b2d21f425f752262f42188365d7b8ff1aff74c82e45136510a4857" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1426,6 +1944,20 @@ dependencies = [ "unicode-xid", ] +[[package]] +name = "procfs" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab8809e0c18450a2db0f236d2a44ec0b4c1412d0eb936233579f0990faa5d5cd" +dependencies = [ + "bitflags", + "byteorder", + "flate2", + "hex", + "lazy_static", + "libc", +] + [[package]] name = "prometheus" version = "0.7.0" @@ -1440,12 +1972,67 @@ dependencies = [ "spin", ] +[[package]] +name = "prometheus" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8425533e7122f0c3cc7a37e6244b16ad3a2cc32ae7ac6276e2a75da0d9c200d" +dependencies = [ + "cfg-if 1.0.0", + "fnv", + "lazy_static", + "libc", + "parking_lot", + "procfs", + "protobuf", + "regex", + "thiserror", +] + [[package]] name = "protobuf" version = "2.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "da78e04bc0e40f36df43ecc6575e4f4b180e8156c4efd73f13d5619479b05696" +[[package]] +name = "pyo3" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf6bbbe8f70d179260b3728e5d04eb012f4f0c7988e58c11433dd689cecaa72e" +dependencies = [ + "ctor", + "indoc", + "inventory", + "libc", + "parking_lot", + "paste", + "pyo3cls", + "unindent", +] + +[[package]] +name = "pyo3-derive-backend" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "10ecd0eb6ed7b3d9965b4f4370b5b9e99e3e5e8742000e1c452c018f8c2a322f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pyo3cls" +version = "0.12.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d344fdaa6a834a06dd1720ff104ea12fe101dad2e8db89345af9db74c0bb11a0" +dependencies = [ + "pyo3-derive-backend", + "quote", + "syn", +] + [[package]] name = "quick-error" version = "1.2.3" @@ -1461,6 +2048,27 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "r2d2" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_postgres" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707d27f66f43bac1081141f6d9611fffcce7da2841ae97c7ac53619d098efe8f" +dependencies = [ + "postgres 0.17.5", + "r2d2", +] + [[package]] name = "rand" version = "0.6.5" @@ -1475,7 +2083,7 @@ dependencies = [ "rand_isaac", "rand_jitter", "rand_os", - "rand_pcg", + "rand_pcg 0.1.2", "rand_xorshift", "winapi 0.3.9", ] @@ -1491,6 +2099,7 @@ dependencies = [ "rand_chacha 0.2.2", "rand_core 0.5.1", "rand_hc 0.2.0", + "rand_pcg 0.2.1", ] [[package]] @@ -1599,6 +2208,15 @@ dependencies = [ "rand_core 0.4.2", ] +[[package]] +name = "rand_pcg" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429" +dependencies = [ + "rand_core 0.5.1", +] + [[package]] name = "rand_xorshift" version = "0.1.1" @@ -1654,7 +2272,10 @@ version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38cf2c13ed4745de91a5eb834e11c00bcc3709e773173b2ce4c56c9fbde04b9c" dependencies = [ + "aho-corasick", + "memchr", "regex-syntax", + "thread_local", ] [[package]] @@ -1682,6 +2303,42 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "reqwest" +version = "0.10.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb15d6255c792356a0f578d8a645c677904dc02e862bebe2ecc18e0c01b9a0ce" +dependencies = [ + "base64 0.13.0", + "bytes 0.5.6", + "encoding_rs", + "futures-core", + "futures-util", + "http", + "http-body", + "hyper", + "hyper-tls", + "ipnet", + "js-sys", + "lazy_static", + "log", + "mime", + "mime_guess", + "native-tls", + "percent-encoding", + "pin-project-lite 0.2.0", + "serde", + "serde_urlencoded 0.7.0", + "tokio 0.2.23", + "tokio-tls", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test", + "web-sys", + "winreg", +] + [[package]] name = "rustc-hash" version = "1.1.0" @@ -1731,6 +2388,25 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ef703b7cb59335eae2eb93ceb664c0eb7ea6bf567079d843e09420219668e072" +[[package]] +name = "schannel" +version = "0.1.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f05ba609c234e60bee0d547fe94a4c7e9da733d1c962cf6e59efa4cd9c8bc75" +dependencies = [ + "lazy_static", + "winapi 0.3.9", +] + +[[package]] +name = "scheduled-thread-pool" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7" +dependencies = [ + "parking_lot", +] + [[package]] name = "scoped-tls" version = "1.0.0" @@ -1749,6 +2425,65 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" +[[package]] +name = "scraper" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48e02aa790c80c2e494130dec6a522033b6a23603ffc06360e9fe6c611ea2c12" +dependencies = [ + "cssparser", + "ego-tree", + "getopts", + "html5ever", + "matches", + "selectors", + "smallvec", + "tendril", +] + +[[package]] +name = "security-framework" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1759c2e3c8580017a484a7ac56d3abc5a6c1feadf88db2f3633f12ae4268c69" +dependencies = [ + "bitflags", + "core-foundation", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f99b9d5e26d2a71633cc4f2ebae7cc9f874044e0c351a27e17892d76dce5678b" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "selectors" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe" +dependencies = [ + "bitflags", + "cssparser", + "derive_more", + "fxhash", + "log", + "matches", + "phf", + "phf_codegen", + "precomputed-hash", + "servo_arc", + "smallvec", + "thin-slice", +] + [[package]] name = "semver" version = "0.9.0" @@ -1807,6 +2542,28 @@ dependencies = [ "url", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "servo_arc" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432" +dependencies = [ + "nodrop", + "stable_deref_trait", +] + [[package]] name = "sha-1" version = "0.8.2" @@ -1906,12 +2663,43 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d" +[[package]] +name = "stable_deref_trait" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" + [[package]] name = "strength_reduce" version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a3ff2f71c82567c565ba4b3009a9350a96a7269eaa4001ebedae926230bc2254" +[[package]] +name = "string_cache" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb1139b5353f96e429e1a5e19fbaf663bddedaa06d1dbd49f82e352601209a" +dependencies = [ + "lazy_static", + "new_debug_unreachable", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f24c8e5e19d22a726626f1a5e16fe15b132dcf21d10177fa5a45ce7962996b97" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + [[package]] name = "stringprep" version = "0.1.2" @@ -1953,6 +2741,43 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tendril" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "707feda9f2582d5d680d733e38755547a3e8fb471e7ba11452ecfd9ce93a5d3b" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] +name = "thin-slice" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c" + +[[package]] +name = "thiserror" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e9ae34b84616eedaaf1e9dd6026dbe00dcafa92aa0c8077cb69df1fcfe5e53e" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ba20f23e85b10754cd195504aebf6a27e2e6cbe28c17778a0c930724628dd56" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "thread_local" version = "1.0.1" @@ -2027,14 +2852,14 @@ version = "0.2.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a6d7ad61edd59bfcc7e80dababf0f4aed2e6d5e0ba1659356ae889752dfc12ff" dependencies = [ - "bytes", + "bytes 0.5.6", "fnv", "futures-core", "iovec", "lazy_static", "libc", "memchr", - "mio", + "mio 0.6.23", "mio-named-pipes", "mio-uds", "num_cpus", @@ -2045,6 +2870,23 @@ dependencies = [ "winapi 0.3.9", ] +[[package]] +name = "tokio" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12a3eb39ee2c231be64487f1fcbe726c8f2514876a55480a5ab8559fc374252" +dependencies = [ + "autocfg 1.0.1", + "bytes 0.6.0", + "futures-core", + "lazy_static", + "libc", + "memchr", + "mio 0.7.6", + "pin-project-lite 0.2.0", + "slab", +] + [[package]] name = "tokio-macros" version = "0.2.6" @@ -2064,7 +2906,7 @@ checksum = "55a2482c9fe4dd481723cf5c0616f34afc710e55dcda0944e12e7b3316117892" dependencies = [ "async-trait", "byteorder", - "bytes", + "bytes 0.5.6", "fallible-iterator", "futures", "log", @@ -2074,8 +2916,40 @@ dependencies = [ "pin-project-lite 0.1.11", "postgres-protocol", "postgres-types", - "tokio", - "tokio-util", + "tokio 0.2.23", + "tokio-util 0.3.1", +] + +[[package]] +name = "tokio-postgres" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "150d9be163b0df6dc185b8ee33bcb9a74865f0cad754495847f2e06e2051a345" +dependencies = [ + "async-trait", + "byteorder", + "bytes 0.5.6", + "fallible-iterator", + "futures", + "log", + "parking_lot", + "percent-encoding", + "phf", + "pin-project-lite 0.1.11", + "postgres-protocol", + "postgres-types", + "tokio 0.3.5", + "tokio-util 0.4.0", +] + +[[package]] +name = "tokio-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a70f4fcd7b3b24fb194f837560168208f669ca8cb70d0c4b862944452396343" +dependencies = [ + "native-tls", + "tokio 0.2.23", ] [[package]] @@ -2087,7 +2961,7 @@ dependencies = [ "futures-util", "log", "pin-project 0.4.27", - "tokio", + "tokio 0.2.23", "tungstenite", ] @@ -2097,12 +2971,26 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" dependencies = [ - "bytes", + "bytes 0.5.6", "futures-core", "futures-sink", "log", "pin-project-lite 0.1.11", - "tokio", + "tokio 0.2.23", +] + +[[package]] +name = "tokio-util" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24793699f4665ba0416ed287dc794fe6b11a4aa5e4e95b58624f45f6c46b97d4" +dependencies = [ + "bytes 0.5.6", + "futures-core", + "futures-sink", + "log", + "pin-project-lite 0.1.11", + "tokio 0.3.5", ] [[package]] @@ -2241,7 +3129,7 @@ checksum = "f0308d80d86700c5878b9ef6321f020f29b1bb9d5ff3cab25e75e23f3a492a23" dependencies = [ "base64 0.12.3", "byteorder", - "bytes", + "bytes 0.5.6", "http", "httparse", "input_buffer", @@ -2294,12 +3182,24 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-width" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9337591893a19b88d8d87f2cec1e73fad5cdfd10e5a6f349f498ad6ea2ffb1e3" + [[package]] name = "unicode-xid" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +[[package]] +name = "unindent" +version = "0.1.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7" + [[package]] name = "url" version = "2.2.0" @@ -2352,7 +3252,7 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f41be6df54c97904af01aa23e613d4521eed7ab23537cede692d4058f6449407" dependencies = [ - "bytes", + "bytes 0.5.6", "futures", "headers", "http", @@ -2365,8 +3265,8 @@ dependencies = [ "scoped-tls", "serde", "serde_json", - "serde_urlencoded", - "tokio", + "serde_urlencoded 0.6.1", + "tokio 0.2.23", "tokio-tungstenite", "tower-service", "tracing", @@ -2386,6 +3286,108 @@ version = "0.10.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a143597ca7c7793eff794def352d41792a93c481eb1042423ff7ff72ba2c31f" +[[package]] +name = "wasm-bindgen" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cd364751395ca0f68cafb17666eee36b63077fb5ecd972bbcd74c90c4bf736e" +dependencies = [ + "cfg-if 1.0.0", + "serde", + "serde_json", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1114f89ab1f4106e5b55e688b828c0ab0ea593a1ea7c094b141b14cbaaec2d62" +dependencies = [ + "bumpalo", + "lazy_static", + "log", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fe9756085a84584ee9457a002b7cdfe0bfff169f45d2591d8be1345a6780e35" +dependencies = [ + "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a6ac8995ead1f084a8dea1e65f194d0973800c7f571f6edd70adf06ecf77084" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a48c72f299d80557c7c62e37e7225369ecc0c963964059509fbafe917c7549" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e7811dd7f9398f14cc76efd356f98f03aa30419dea46aa810d71e819fc97158" + +[[package]] +name = "wasm-bindgen-test" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0355fa0c1f9b792a09b6dcb6a8be24d51e71e6d74972f9eb4a44c4c004d24a25" +dependencies = [ + "console_error_panic_hook", + "js-sys", + "scoped-tls", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-bindgen-test-macro", +] + +[[package]] +name = "wasm-bindgen-test-macro" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27e07b46b98024c2ba2f9e83a10c2ef0515f057f2da299c1762a2017de80438b" +dependencies = [ + "proc-macro2", + "quote", +] + +[[package]] +name = "web-sys" +version = "0.3.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "222b1ef9334f92a21d3fb53dc3fd80f30836959a90f9274a626d7e06315ba3c3" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "weezl" version = "0.1.3" @@ -2426,6 +3428,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "winreg" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0120db82e8a1e0b9fb3345a539c478767c0048d842860994d96113d5b667bd69" +dependencies = [ + "winapi 0.3.9", +] + [[package]] name = "ws2_32-sys" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index dd0af91..55954ff 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,7 +1,8 @@ [workspace] members = [ "fuzzysearch", - "fuzzysearch-common" + "fuzzysearch-common", + "fuzzysearch-ingest-furaffinity" ] [profile.dev] diff --git a/fuzzysearch-ingest-furaffinity/Cargo.toml b/fuzzysearch-ingest-furaffinity/Cargo.toml index 3db1e33..7b215ac 100644 --- a/fuzzysearch-ingest-furaffinity/Cargo.toml +++ b/fuzzysearch-ingest-furaffinity/Cargo.toml @@ -15,7 +15,6 @@ chrono = "0.4" hyper = "0.13" prometheus = { version = "0.11", features = ["process"] } lazy_static = "1" -hex = "0.4" [dependencies.furaffinity-rs] git = "https://github.com/Syfaro/furaffinity-rs" diff --git a/fuzzysearch-ingest-furaffinity/src/main.rs b/fuzzysearch-ingest-furaffinity/src/main.rs index da43db5..48eb1c3 100644 --- a/fuzzysearch-ingest-furaffinity/src/main.rs +++ b/fuzzysearch-ingest-furaffinity/src/main.rs @@ -73,7 +73,7 @@ async fn ids_to_check(client: &Client, max: i32) -> Vec { async fn insert_submission( client: &Client, sub: &furaffinity_rs::Submission, -) -> Result<(), postgres::Error> { +) -> Result<(), tokio_postgres::Error> { let artist_id = lookup_artist(&client, &sub.artist).await; let mut tag_ids = Vec::with_capacity(sub.tags.len()); for tag in &sub.tags { @@ -100,7 +100,7 @@ async fn insert_submission( Ok(()) } -async fn insert_null_submission(client: &Client, id: i32) -> Result { +async fn insert_null_submission(client: &Client, id: i32) -> Result { client .execute("INSERT INTO SUBMISSION (id) VALUES ($1)", &[&id]) .await @@ -153,9 +153,6 @@ async fn main() { std::env::var("FA_B").expect("missing fa cookie b"), ); - let path = std::env::var("OUT_DIR").expect("missing output directory"); - let path = std::path::Path::new(&path); - let user_agent = std::env::var("USER_AGENT").expect("missing user agent"); let fa = furaffinity_rs::FurAffinity::new(cookie_a, cookie_b, user_agent); @@ -223,26 +220,6 @@ async fn main() { timer.stop_and_record(); - if let Some(sha) = &sub.file_sha256 { - use tokio::io::AsyncWriteExt; - - let file = sub.file.as_ref().unwrap(); - let ext = sub.filename.split('.').last().unwrap(); - - let h = hex::encode(sha); - let p = path.join(&h[0..2]).join(&h[2..4]); - std::fs::create_dir_all(&p).expect("unable to create hash directory"); - - let name = format!("{}.{}", h, ext); - let name = std::path::Path::new(&name); - let name = p.join(name); - - if !name.exists() { - let mut f = tokio::fs::File::create(&name).await.expect("unable to create submission file"); - f.write_all(file).await.expect("unable to write file contents"); - } - } - insert_submission(&client, &sub).await.unwrap(); break 'attempt;