Update package names and Dockerfiles.

Cargo.lock

@@ -509,27 +509,6 @@ dependencies = [
  "dtoa",
 ]
 
-[[package]]
-name = "e621-watcher"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "hamming",
- "hyper",
- "image",
- "img_hash",
- "lazy_static",
- "prometheus 0.11.0",
- "reqwest",
- "serde",
- "serde_json",
- "sha2",
- "sqlx",
- "tokio",
- "tracing",
- "tracing-subscriber",
-]
-
 [[package]]
 name = "ego-tree"
 version = "0.6.2"
@@ -807,6 +786,28 @@ dependencies = [
  "tracing",
 ]
 
+[[package]]
+name = "fuzzysearch-ingest-e621"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "fuzzysearch-common",
+ "hamming",
+ "hyper",
+ "image",
+ "img_hash",
+ "lazy_static",
+ "prometheus 0.11.0",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "sha2",
+ "sqlx",
+ "tokio",
+ "tracing",
+ "tracing-subscriber",
+]
+
 [[package]]
 name = "fuzzysearch-ingest-furaffinity"
 version = "0.1.0"
@@ -824,6 +825,22 @@ dependencies = [
  "tokio-postgres",
 ]
 
+[[package]]
+name = "fuzzysearch-ingest-weasyl"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "fuzzysearch-common",
+ "image",
+ "img_hash",
+ "reqwest",
+ "serde",
+ "serde_json",
+ "sha2",
+ "sqlx",
+ "tokio",
+]
+
 [[package]]
 name = "fxhash"
 version = "0.2.1"
@@ -3362,22 +3379,6 @@ version = "0.2.70"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64"
 
-[[package]]
-name = "weasyl-watcher"
-version = "0.1.0"
-dependencies = [
- "anyhow",
- "fuzzysearch-common",
- "image",
- "img_hash",
- "reqwest",
- "serde",
- "serde_json",
- "sha2",
- "sqlx",
- "tokio",
-]
-
 [[package]]
 name = "web-sys"
 version = "0.3.47"

fuzzysearch-common/Cargo.toml

@@ -4,6 +4,11 @@ version = "0.1.0"
 authors = ["Syfaro <syfaro@huefox.com>"]
 edition = "2018"
 
+[features]
+default = []
+
+video = ["ffmpeg-next", "tempfile"]
+
 [dependencies]
 anyhow = "1"
 tracing = "0.1"
@@ -13,5 +18,5 @@ serde = { version = "1", features = ["derive"] }
 image = "0.23"
 img_hash = "3"
 
-ffmpeg-next = "4"
+ffmpeg-next = { version = "4", optional = true }
-tempfile = "3"
+tempfile = { version = "3", optional = true }

fuzzysearch-common/src/lib.rs

@@ -1,4 +1,5 @@
 pub mod types;
+#[cfg(feature = "video")]
 pub mod video;
 
 /// Create an instance of img_hash with project defaults.

fuzzysearch-ingest-e621/.dockerignore

@@ -1 +0,0 @@
-target/

fuzzysearch-ingest-e621/.drone.yml

@@ -1,25 +0,0 @@
----
-kind: pipeline
-type: docker
-name: default
-
-platform:
-  os: linux
-  arch: amd64
-
-steps:
-- name: build-latest
-  image: plugins/docker
-  settings:
-    auto_tag: true
-    password:
-      from_secret: docker_password
-    registry: registry.huefox.com
-    repo: registry.huefox.com/e621-watcher
-    username:
-      from_secret: docker_username
-  when:
-    branch:
-    - master
-
-...

fuzzysearch-ingest-e621/.gitignore

@@ -1,2 +0,0 @@
-/target
-.env

fuzzysearch-ingest-e621/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "e621-watcher"
+name = "fuzzysearch-ingest-e621"
 version = "0.1.0"
 authors = ["Syfaro <syfaro@huefox.com>"]
 edition = "2018"
@@ -27,3 +27,5 @@ anyhow = "1"
 
 lazy_static = "1"
 prometheus = { version = "0.11", features = ["process"] }
+
+fuzzysearch-common = { path = "../fuzzysearch-common" }

fuzzysearch-ingest-e621/Dockerfile

@@ -1,14 +1,14 @@
-FROM rust:1-slim AS builder
+FROM rust:1-slim-buster AS builder
 WORKDIR /src
 ENV SQLX_OFFLINE=true
 RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
 COPY . .
-RUN cargo install --root / --path .
+RUN cargo install --root / --path ./fuzzysearch-ingest-e621
 
 FROM debian:buster-slim
 EXPOSE 8080
 ENV METRICS_HOST=0.0.0.0:8080
 WORKDIR /app
 RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/*
-COPY --from=builder /bin/e621-watcher /bin/e621-watcher
+COPY --from=builder /bin/fuzzysearch-ingest-e621 /bin/fuzzysearch-ingest-e621
-CMD ["/bin/e621-watcher"]
+CMD ["/bin/fuzzysearch-ingest-e621"]

fuzzysearch-ingest-e621/src/main.rs

@@ -272,7 +272,7 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result<Image
 
     tracing::trace!(?result, "Calculated image SHA256");
 
-    let hasher = get_hasher();
+    let hasher = fuzzysearch_common::get_hasher();
     let img = match image::load_from_memory(&bytes) {
         Ok(img) => img,
         Err(err) => {
@@ -292,14 +292,6 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result<Image
     Ok((Some(hash), None, Some(result)))
 }
 
-fn get_hasher() -> img_hash::Hasher<[u8; 8]> {
-    img_hash::HasherConfig::with_bytes_type::<[u8; 8]>()
-        .hash_alg(img_hash::HashAlg::Gradient)
-        .hash_size(8, 8)
-        .preproc_dct()
-        .to_hasher()
-}
-
 async fn provide_metrics(
     _: hyper::Request<hyper::Body>,
 ) -> Result<hyper::Response<hyper::Body>, std::convert::Infallible> {

fuzzysearch-ingest-furaffinity/Cargo.toml

@@ -12,7 +12,7 @@ tokio-postgres = { version = "0.7.0" }
 r2d2_postgres = " 0.18.0"
 r2d2 = "0.8"
 chrono = "0.4"
-hyper = "0.14"
+hyper = { version = "0.14", features = ["server"] }
 prometheus = { version = "0.11", features = ["process"] }
 lazy_static = "1"
 

fuzzysearch-ingest-furaffinity/Dockerfile

@@ -1,5 +1,14 @@
-FROM rustlang/rust:nightly-slim
+FROM rust:1-slim-buster AS builder
+WORKDIR /src
+ENV SQLX_OFFLINE=true
+RUN apt-get update -y && apt-get install -y libssl-dev pkg-config python3 python3-pip
 COPY . .
-RUN apt-get -y update && apt-get -y install pkg-config libssl-dev
+RUN cargo install --root / --path ./fuzzysearch-ingest-furaffinity
-RUN cargo install --root / --path .
+
-CMD ["/bin/fa-watcher"]
+FROM rustlang/rust:nightly-slim
+RUN apt-get update -y && \
+    apt-get install -y openssl ca-certificates python3 python3-pip && \
+    python3 -m pip --no-cache-dir install cfscrape && \
+    rm -rf /var/lib/apt/lists/*
+COPY --from=builder /bin/fuzzysearch-ingest-furaffinity /bin/fuzzysearch-ingest-furaffinity
+CMD ["/bin/fuzzysearch-ingest-furaffinity"]

fuzzysearch-ingest-weasyl/Cargo.toml

@@ -1,5 +1,5 @@
 [package]
-name = "weasyl-watcher"
+name = "fuzzysearch-ingest-weasyl"
 version = "0.1.0"
 authors = ["Syfaro <syfaro@huefox.com>"]
 edition = "2018"

fuzzysearch-ingest-weasyl/Dockerfile

@@ -1,4 +1,11 @@
-FROM debian:buster-slim
+FROM rust:1-slim-buster AS builder
-RUN apt-get update -y && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/*
+WORKDIR /src
-COPY ./weasyl-watcher /bin/weasyl-watcher
+ENV SQLX_OFFLINE=true
-CMD ["/bin/weasyl-watcher"]
+RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
+COPY . .
+RUN cargo install --root / --path ./fuzzysearch-ingest-weasyl
+
+FROM rustlang/rust:nightly-slim
+RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/*
+COPY --from=builder /bin/fuzzysearch-ingest-weasyl /bin/fuzzysearch-ingest-weasyl
+CMD ["/bin/fuzzysearch-ingest-weasyl"]

fuzzysearch/Cargo.toml

@@ -41,4 +41,4 @@ hamming = "0.1"
 
 bk-tree = "0.3"
 
-fuzzysearch-common = { path = "../fuzzysearch-common" }
+fuzzysearch-common = { path = "../fuzzysearch-common", features = ["video"] }

fuzzysearch/Dockerfile

@@ -1,9 +1,9 @@
-FROM rust:1-slim AS builder
+FROM rust:1-slim-buster AS builder
 WORKDIR /src
 ENV SQLX_OFFLINE=true
-RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
+RUN apt-get update -y && apt-get install -y libssl-dev pkg-config libavcodec-dev libavformat-dev libavutil-dev libavdevice-dev clang llvm
 COPY . .
-RUN cargo install --root / --path .
+RUN cargo install --root / --path ./fuzzysearch
 
 FROM debian:buster-slim
 EXPOSE 8080 8081