anthrove/fuzzysearch
4
0
Fork 0
mirror of https://github.com/Syfaro/fuzzysearch.git synced 2024-11-23 15:22:31 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update Weasyl ingester.

Browse Source
This commit is contained in:
Syfaro 2020-12-07 18:17:38 -06:00
parent 1fda51cb5d
commit 415d030bd2
8 changed files with 444 additions and 2074 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

421
Cargo.lock generated
View File

@ -12,6 +12,16 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234"
[[package]]
name = "ahash"
version = "0.5.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "062782ee945b38cab67201cd84c77ab3881c07125a1f1f7a5d2fd213f797a2d6"
dependencies = [
"getrandom 0.2.0",
"lazy_static",
]
[[package]]
name = "aho-corasick"
version = "0.7.15"
@ -36,6 +46,12 @@ version = "1.0.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c0df63cb2955042487fad3aefd2c6e3ae7389ac5dc1beb28921de0b69f779d4"
[[package]]
name = "arrayvec"
version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "23b62fc65de8e4e7f52534fb52b0f3ed04746ae267519eef2a83941e8085068b"
[[package]]
name = "async-trait"
version = "0.1.42"
@ -47,6 +63,15 @@ dependencies = [
"syn",
]
[[package]]
name = "atoi"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c897df197d57c25b37df9d8fa2f93ddbfeee9ebd2264350ac79c8ec4b795885"
dependencies = [
"num-traits",
]
[[package]]
name = "autocfg"
version = "0.1.7"
@ -127,6 +152,18 @@ version = "1.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"
[[package]]
name = "bitvec"
version = "0.19.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a7ba35e9565969edb811639dbebfe34edc0368e472c5018474c8eb2543397f81"
dependencies = [
"funty",
"radium",
"tap",
"wyz",
]
[[package]]
name = "bk-tree"
version = "0.3.0"
@ -221,7 +258,7 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f4aedb84272dbe89af497cf81375129abda4fc0a9e7c5d317498c15cc30c0d27"
dependencies = [
"nom",
"nom 5.1.2",
]
[[package]]
@ -330,6 +367,16 @@ dependencies = [
"cfg-if 1.0.0",
]
[[package]]
name = "crossbeam-channel"
version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87"
dependencies = [
"crossbeam-utils 0.7.2",
"maybe-uninit",
]
[[package]]
name = "crossbeam-channel"
version = "0.5.0"
@ -337,7 +384,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dca26ee1f8d361640700bde38b2c37d8c22b3ce2d360e1fc1c74ea4b0aa7d775"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-utils",
"crossbeam-utils 0.8.1",
]
[[package]]
@ -348,7 +395,7 @@ checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
dependencies = [
"cfg-if 1.0.0",
"crossbeam-epoch",
"crossbeam-utils",
"crossbeam-utils 0.8.1",
]
[[package]]
@ -359,12 +406,34 @@ checksum = "a1aaa739f95311c2c7887a76863f500026092fb1dce0161dab577e559ef3569d"
dependencies = [
"cfg-if 1.0.0",
"const_fn",
"crossbeam-utils",
"crossbeam-utils 0.8.1",
"lazy_static",
"memoffset",
"scopeguard",
]
[[package]]
name = "crossbeam-queue"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570"
dependencies = [
"cfg-if 0.1.10",
"crossbeam-utils 0.7.2",
"maybe-uninit",
]
[[package]]
name = "crossbeam-utils"
version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8"
dependencies = [
"autocfg 1.0.1",
"cfg-if 0.1.10",
"lazy_static",
]
[[package]]
name = "crossbeam-utils"
version = "0.8.1"
@ -462,6 +531,12 @@ dependencies = [
"generic-array 0.14.4",
]
[[package]]
name = "dotenv"
version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f"
[[package]]
name = "dtoa"
version = "0.4.6"
@ -488,6 +563,9 @@ name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
dependencies = [
"serde",
]
[[package]]
name = "encoding_rs"
@ -600,6 +678,12 @@ version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7"
[[package]]
name = "funty"
version = "1.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ba62103ce691c2fd80fbae2213dfdda9ce60804973ac6b6e97de818ea7f52c8"
[[package]]
name = "furaffinity-rs"
version = "0.1.0"
@ -843,6 +927,17 @@ dependencies = [
"wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
name = "getrandom"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee8025cf36f917e6a52cce185b7c7177689b838b7ec138364e50cc2277a56cf4"
dependencies = [
"cfg-if 0.1.10",
"libc",
"wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
name = "ghost"
version = "0.1.2"
@ -927,6 +1022,15 @@ dependencies = [
"http",
]
[[package]]
name = "heck"
version = "0.3.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "20564e78d53d2bb135c343b3f47714a56af2061f1c928fdb541dc7b9fdd94205"
dependencies = [
"unicode-segmentation",
]
[[package]]
name = "hermit-abi"
version = "0.1.17"
@ -1226,6 +1330,19 @@ version = "1.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
[[package]]
name = "lexical-core"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "db65c6da02e61f55dae90a0ae427b2a5f6b3e8db09f58d10efab23af92592616"
dependencies = [
"arrayvec",
"bitflags",
"cfg-if 0.1.10",
"ryu",
"static_assertions",
]
[[package]]
name = "libc"
version = "0.2.81"
@ -1242,6 +1359,12 @@ dependencies = [
"winapi 0.3.9",
]
[[package]]
name = "linked-hash-map"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8dd5a6d5999d9907cda8ed67bbd137d3af8085216c2ac62de5be860bd41f304a"
[[package]]
name = "lock_api"
version = "0.4.2"
@ -1273,12 +1396,27 @@ dependencies = [
"serde_json",
]
[[package]]
name = "lru-cache"
version = "0.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "31e24f1ad8321ca0e8a1e0ac13f23cb668e6f5466c2c57319f6a5cf1cc8e3b1c"
dependencies = [
"linked-hash-map",
]
[[package]]
name = "mac"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
[[package]]
name = "maplit"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
[[package]]
name = "markup5ever"
version = "0.10.0"
@ -1311,6 +1449,23 @@ version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"
[[package]]
name = "maybe-uninit"
version = "2.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00"
[[package]]
name = "md-5"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"
dependencies = [
"block-buffer 0.9.0",
"digest 0.9.0",
"opaque-debug 0.3.0",
]
[[package]]
name = "md5"
version = "0.7.0"
@ -1513,6 +1668,18 @@ dependencies = [
"version_check",
]
[[package]]
name = "nom"
version = "6.0.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "88034cfd6b4a0d54dd14f4a507eceee36c0b70e5a02236c4e4df571102be17f0"
dependencies = [
"bitvec",
"lexical-core",
"memchr",
"version_check",
]
[[package]]
name = "ntapi"
version = "0.3.6"
@ -2069,6 +2236,12 @@ dependencies = [
"r2d2",
]
[[package]]
name = "radium"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "941ba9d78d8e2f7ce474c015eea4d9c6d25b6a3327f9832ee29a4de27f91bbb8"
[[package]]
name = "rand"
version = "0.6.5"
@ -2094,7 +2267,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
"getrandom",
"getrandom 0.1.15",
"libc",
"rand_chacha 0.2.2",
"rand_core 0.5.1",
@ -2143,7 +2316,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
"getrandom",
"getrandom 0.1.15",
]
[[package]]
@ -2244,9 +2417,9 @@ version = "1.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9ab346ac5921dc62ffa9f89b7a773907511cdfa5490c572ae9be1be33e8afa4a"
dependencies = [
"crossbeam-channel",
"crossbeam-channel 0.5.0",
"crossbeam-deque",
"crossbeam-utils",
"crossbeam-utils 0.8.1",
"lazy_static",
"num_cpus",
]
@ -2328,6 +2501,7 @@ dependencies = [
"percent-encoding",
"pin-project-lite 0.2.0",
"serde",
"serde_json",
"serde_urlencoded 0.7.0",
"tokio 0.2.23",
"tokio-tls",
@ -2339,6 +2513,21 @@ dependencies = [
"winreg",
]
[[package]]
name = "ring"
version = "0.16.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "024a1e66fea74c66c66624ee5622a7ff0e4b73a13b4f5c326ddb50c708944226"
dependencies = [
"cc",
"libc",
"once_cell",
"spin",
"untrusted",
"web-sys",
"winapi 0.3.9",
]
[[package]]
name = "rustc-hash"
version = "1.1.0"
@ -2376,6 +2565,19 @@ dependencies = [
"transpose 0.1.0",
]
[[package]]
name = "rustls"
version = "0.18.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5d1126dcf58e93cee7d098dbda643b5f92ed724f1f6a63007c1116eed6700c81"
dependencies = [
"base64 0.12.3",
"log",
"ring",
"sct",
"webpki",
]
[[package]]
name = "ryu"
version = "1.0.5"
@ -2441,6 +2643,16 @@ dependencies = [
"tendril",
]
[[package]]
name = "sct"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3042af939fca8c3453b7af0f1c66e533a15a86169e39de2657310ade8f98d3c"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "security-framework"
version = "2.0.0"
@ -2525,6 +2737,7 @@ version = "1.0.60"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1500e84d27fe482ed1dc791a56eddc2f230046a040fa908c08bda1d9fb615779"
dependencies = [
"indexmap",
"itoa",
"ryu",
"serde",
@ -2663,12 +2876,121 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6e63cff320ae2c57904679ba7cb63280a3dc4613885beafb148ee7bf9aa9042d"
[[package]]
name = "sqlformat"
version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74c70f0235b9925cbb106c52af1a28b5ea4885a8b851e328b8562e257a389c2d"
dependencies = [
"lazy_static",
"maplit",
"nom 6.0.1",
"regex",
"unicode_categories",
]
[[package]]
name = "sqlx"
version = "0.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d1f8eb788e1733bdbf69a8f97087213ebdebd253d4782c686d3cfd586b0a9453"
dependencies = [
"sqlx-core",
"sqlx-macros",
]
[[package]]
name = "sqlx-core"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5e647268dc1239dd9db2d3103fefd61151971a2214882cff9efea6f60cf50840"
dependencies = [
"ahash",
"atoi",
"base64 0.13.0",
"bitflags",
"byteorder",
"bytes 0.5.6",
"crossbeam-channel 0.4.4",
"crossbeam-queue",
"crossbeam-utils 0.7.2",
"either",
"futures-channel",
"futures-core",
"futures-util",
"hex",
"hmac",
"itoa",
"libc",
"log",
"lru-cache",
"md-5",
"memchr",
"once_cell",
"parking_lot",
"percent-encoding",
"rand 0.7.3",
"rustls",
"serde",
"serde_json",
"sha-1 0.9.2",
"sha2",
"smallvec",
"sqlformat",
"sqlx-rt",
"stringprep",
"thiserror",
"url",
"webpki",
"webpki-roots",
"whoami",
]
[[package]]
name = "sqlx-macros"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c7acd32cba35531345f8a94a038874baf00efd0b701c913f5b00d2870b474b64"
dependencies = [
"dotenv",
"either",
"futures",
"heck",
"hex",
"proc-macro2",
"quote",
"serde",
"serde_json",
"sha2",
"sqlx-core",
"sqlx-rt",
"syn",
"url",
]
[[package]]
name = "sqlx-rt"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63fc5454c9dd7aaea3a0eeeb65ca40d06d0d8e7413a8184f7c3a3ffa5056190b"
dependencies = [
"once_cell",
"tokio 0.2.23",
"tokio-rustls",
]
[[package]]
name = "stable_deref_trait"
version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "static_assertions"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
[[package]]
name = "strength_reduce"
version = "0.2.3"
@ -2727,6 +3049,12 @@ dependencies = [
"unicode-xid",
]
[[package]]
name = "tap"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "36474e732d1affd3a6ed582781b3683df3d0563714c59c39591e8ff707cf078e"
[[package]]
name = "tempfile"
version = "3.1.0"
@ -2942,6 +3270,18 @@ dependencies = [
"tokio-util 0.4.0",
]
[[package]]
name = "tokio-rustls"
version = "0.14.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e12831b255bcfa39dc0436b01e19fea231a37db570686c06ee72c423479f889a"
dependencies = [
"futures-core",
"rustls",
"tokio 0.2.23",
"webpki",
]
[[package]]
name = "tokio-tls"
version = "0.3.1"
@ -3182,6 +3522,12 @@ dependencies = [
"tinyvec",
]
[[package]]
name = "unicode-segmentation"
version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb0d2e7be6ae3a5fa87eed5fb451aff96f2573d2694942e40543ae0bbe19c796"
[[package]]
name = "unicode-width"
version = "0.1.8"
@ -3194,12 +3540,24 @@ version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564"
[[package]]
name = "unicode_categories"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e"
[[package]]
name = "unindent"
version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f14ee04d9415b52b3aeab06258a3f07093182b88ba0f9b8d203f211a7a7d41c7"
[[package]]
name = "untrusted"
version = "0.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
[[package]]
name = "url"
version = "2.2.0"
@ -3378,6 +3736,22 @@ dependencies = [
"quote",
]
[[package]]
name = "weasyl-watcher"
version = "0.1.0"
dependencies = [
"anyhow",
"fuzzysearch-common",
"image",
"img_hash",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio 0.2.23",
]
[[package]]
name = "web-sys"
version = "0.3.46"
@ -3388,12 +3762,37 @@ dependencies = [
"wasm-bindgen",
]
[[package]]
name = "webpki"
version = "0.21.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea"
dependencies = [
"ring",
"untrusted",
]
[[package]]
name = "webpki-roots"
version = "0.20.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f20dea7535251981a9670857150d571846545088359b28e4951d350bdaf179f"
dependencies = [
"webpki",
]
[[package]]
name = "weezl"
version = "0.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e2bb9fc8309084dd7cd651336673844c1d47f8ef6d2091ec160b27f5c4aa277"
[[package]]
name = "whoami"
version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7884773ab69074615cb8f8425d0e53f11710786158704fca70f53e71b0e05504"
[[package]]
name = "winapi"
version = "0.2.8"
@ -3446,3 +3845,9 @@ dependencies = [
"winapi 0.2.8",
"winapi-build",
]
[[package]]
name = "wyz"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "85e60b0d1b5f99db2556934e21937020776a5d31520bf169e851ac44e6420214"

3
Cargo.toml
View File

@ -2,7 +2,8 @@
members = [
"fuzzysearch",
"fuzzysearch-common",
"fuzzysearch-ingest-furaffinity"
"fuzzysearch-ingest-furaffinity",
"fuzzysearch-ingest-weasyl"
]
[profile.dev]

45
fuzzysearch-ingest-weasyl/.drone.yml
View File

@ -1,45 +0,0 @@
---
kind: pipeline
type: docker
name: default
platform:
os: linux
arch: amd64
steps:
- name: build
pull: always
image: rust:1.47-slim-buster
commands:
- apt-get update -y
- apt-get install -y wget libssl-dev pkg-config
- wget -O sccache.tar.gz https://github.com/mozilla/sccache/releases/download/0.2.13/sccache-0.2.13-x86_64-unknown-linux-musl.tar.gz
- tar zxvf sccache.tar.gz
- export RUSTC_WRAPPER=$(pwd)/sccache-0.2.13-x86_64-unknown-linux-musl/sccache
- cargo build --release
- $(pwd)/sccache-0.2.13-x86_64-unknown-linux-musl/sccache --show-stats
- cp ./target/release/weasyl-watcher ./weasyl-watcher
environment:
AWS_ACCESS_KEY_ID:
from_secret: sccache_s3_access_key
AWS_SECRET_ACCESS_KEY:
from_secret: sccache_s3_secret_key
SCCACHE_BUCKET: cache
SCCACHE_ENDPOINT:
from_secret: sccache_s3_endpoint
SCCACHE_S3_USE_SSL: true
- name: docker
image: plugins/docker
settings:
tags:
- latest
password:
from_secret: docker_password
registry: registry.huefox.com
repo: registry.huefox.com/weasyl-watcher
username:
from_secret: docker_username
...

1
fuzzysearch-ingest-weasyl/.gitignore vendored
View File

@ -1 +0,0 @@
/target

1989
fuzzysearch-ingest-weasyl/Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

11
fuzzysearch-ingest-weasyl/Cargo.toml
View File

@ -18,10 +18,9 @@ img_hash = "3"
sha2 = "0.9"
[dependencies.sqlx]
version = "0.4.0-beta.1"
default-features = false
features = ["runtime-tokio", "macros", "postgres", "json", "offline"]
fuzzysearch-common = { path = "../fuzzysearch-common" }
[profile.dev.package."*"]
opt-level = 2
[dependencies.sqlx]
version = "0.4"
default-features = false
features = ["runtime-tokio-rustls", "macros", "postgres", "json", "offline"]

10
fuzzysearch-ingest-weasyl/src/main.rs
View File

@ -127,11 +127,7 @@ async fn process_submission(
.await?;
let num = if let Ok(image) = image::load_from_memory(&data) {
let hasher = img_hash::HasherConfig::with_bytes_type::<[u8; 8]>()
.hash_alg(img_hash::HashAlg::Gradient)
.hash_size(8, 8)
.preproc_dct()
.to_hasher();
let hasher = fuzzysearch_common::get_hasher();
let hash = hasher.hash_image(&image);
let mut bytes: [u8; 8] = [0; 8];
bytes.copy_from_slice(hash.as_bytes());
@ -187,6 +183,7 @@ async fn main() {
let client = reqwest::Client::new();
loop {
let min = sqlx::query!("SELECT max(id) id FROM weasyl")
.fetch_one(&pool)
.await
@ -210,4 +207,7 @@ async fn main() {
(None, body) => insert_null(&pool, body, id).await.unwrap(),
}
}
tokio::time::delay_for(std::time::Duration::from_secs(60 * 5)).await;
}
}

0
fuzzysearch-ingest-weasyl/migrations/20201009171601_initial-weasyl-table.sql → migrations/20201009171601_initial-weasyl-table.sql
View File