diff --git a/Cargo.lock b/Cargo.lock index e821643..36ed889 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -509,27 +509,6 @@ dependencies = [ "dtoa", ] -[[package]] -name = "e621-watcher" -version = "0.1.0" -dependencies = [ - "anyhow", - "hamming", - "hyper", - "image", - "img_hash", - "lazy_static", - "prometheus 0.11.0", - "reqwest", - "serde", - "serde_json", - "sha2", - "sqlx", - "tokio", - "tracing", - "tracing-subscriber", -] - [[package]] name = "ego-tree" version = "0.6.2" @@ -807,6 +786,28 @@ dependencies = [ "tracing", ] +[[package]] +name = "fuzzysearch-ingest-e621" +version = "0.1.0" +dependencies = [ + "anyhow", + "fuzzysearch-common", + "hamming", + "hyper", + "image", + "img_hash", + "lazy_static", + "prometheus 0.11.0", + "reqwest", + "serde", + "serde_json", + "sha2", + "sqlx", + "tokio", + "tracing", + "tracing-subscriber", +] + [[package]] name = "fuzzysearch-ingest-furaffinity" version = "0.1.0" @@ -824,6 +825,22 @@ dependencies = [ "tokio-postgres", ] +[[package]] +name = "fuzzysearch-ingest-weasyl" +version = "0.1.0" +dependencies = [ + "anyhow", + "fuzzysearch-common", + "image", + "img_hash", + "reqwest", + "serde", + "serde_json", + "sha2", + "sqlx", + "tokio", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -3362,22 +3379,6 @@ version = "0.2.70" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64" -[[package]] -name = "weasyl-watcher" -version = "0.1.0" -dependencies = [ - "anyhow", - "fuzzysearch-common", - "image", - "img_hash", - "reqwest", - "serde", - "serde_json", - "sha2", - "sqlx", - "tokio", -] - [[package]] name = "web-sys" version = "0.3.47" diff --git a/fuzzysearch-common/Cargo.toml b/fuzzysearch-common/Cargo.toml index 2b3367d..e1144d9 100644 --- a/fuzzysearch-common/Cargo.toml +++ b/fuzzysearch-common/Cargo.toml @@ -4,6 +4,11 @@ version = "0.1.0" authors = ["Syfaro "] edition = "2018" +[features] +default = [] + +video = ["ffmpeg-next", "tempfile"] + [dependencies] anyhow = "1" tracing = "0.1" @@ -13,5 +18,5 @@ serde = { version = "1", features = ["derive"] } image = "0.23" img_hash = "3" -ffmpeg-next = "4" -tempfile = "3" +ffmpeg-next = { version = "4", optional = true } +tempfile = { version = "3", optional = true } diff --git a/fuzzysearch-common/src/lib.rs b/fuzzysearch-common/src/lib.rs index 3078aed..0f09586 100644 --- a/fuzzysearch-common/src/lib.rs +++ b/fuzzysearch-common/src/lib.rs @@ -1,4 +1,5 @@ pub mod types; +#[cfg(feature = "video")] pub mod video; /// Create an instance of img_hash with project defaults. diff --git a/fuzzysearch-ingest-e621/.dockerignore b/fuzzysearch-ingest-e621/.dockerignore deleted file mode 100644 index 2f7896d..0000000 --- a/fuzzysearch-ingest-e621/.dockerignore +++ /dev/null @@ -1 +0,0 @@ -target/ diff --git a/fuzzysearch-ingest-e621/.drone.yml b/fuzzysearch-ingest-e621/.drone.yml deleted file mode 100644 index 644af61..0000000 --- a/fuzzysearch-ingest-e621/.drone.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- -kind: pipeline -type: docker -name: default - -platform: - os: linux - arch: amd64 - -steps: -- name: build-latest - image: plugins/docker - settings: - auto_tag: true - password: - from_secret: docker_password - registry: registry.huefox.com - repo: registry.huefox.com/e621-watcher - username: - from_secret: docker_username - when: - branch: - - master - -... diff --git a/fuzzysearch-ingest-e621/.gitignore b/fuzzysearch-ingest-e621/.gitignore deleted file mode 100644 index fedaa2b..0000000 --- a/fuzzysearch-ingest-e621/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -/target -.env diff --git a/fuzzysearch-ingest-e621/Cargo.toml b/fuzzysearch-ingest-e621/Cargo.toml index d780fa1..7254d7a 100644 --- a/fuzzysearch-ingest-e621/Cargo.toml +++ b/fuzzysearch-ingest-e621/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "e621-watcher" +name = "fuzzysearch-ingest-e621" version = "0.1.0" authors = ["Syfaro "] edition = "2018" @@ -27,3 +27,5 @@ anyhow = "1" lazy_static = "1" prometheus = { version = "0.11", features = ["process"] } + +fuzzysearch-common = { path = "../fuzzysearch-common" } diff --git a/fuzzysearch-ingest-e621/Dockerfile b/fuzzysearch-ingest-e621/Dockerfile index 36ee434..9fb4346 100644 --- a/fuzzysearch-ingest-e621/Dockerfile +++ b/fuzzysearch-ingest-e621/Dockerfile @@ -1,14 +1,14 @@ -FROM rust:1-slim AS builder +FROM rust:1-slim-buster AS builder WORKDIR /src ENV SQLX_OFFLINE=true RUN apt-get update -y && apt-get install -y libssl-dev pkg-config COPY . . -RUN cargo install --root / --path . +RUN cargo install --root / --path ./fuzzysearch-ingest-e621 FROM debian:buster-slim EXPOSE 8080 ENV METRICS_HOST=0.0.0.0:8080 WORKDIR /app RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* -COPY --from=builder /bin/e621-watcher /bin/e621-watcher -CMD ["/bin/e621-watcher"] +COPY --from=builder /bin/fuzzysearch-ingest-e621 /bin/fuzzysearch-ingest-e621 +CMD ["/bin/fuzzysearch-ingest-e621"] diff --git a/fuzzysearch-ingest-e621/src/main.rs b/fuzzysearch-ingest-e621/src/main.rs index 0c93de6..1f831f7 100644 --- a/fuzzysearch-ingest-e621/src/main.rs +++ b/fuzzysearch-ingest-e621/src/main.rs @@ -272,7 +272,7 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result img, Err(err) => { @@ -292,14 +292,6 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result img_hash::Hasher<[u8; 8]> { - img_hash::HasherConfig::with_bytes_type::<[u8; 8]>() - .hash_alg(img_hash::HashAlg::Gradient) - .hash_size(8, 8) - .preproc_dct() - .to_hasher() -} - async fn provide_metrics( _: hyper::Request, ) -> Result, std::convert::Infallible> { diff --git a/fuzzysearch-ingest-furaffinity/Cargo.toml b/fuzzysearch-ingest-furaffinity/Cargo.toml index 9314281..95fd31e 100644 --- a/fuzzysearch-ingest-furaffinity/Cargo.toml +++ b/fuzzysearch-ingest-furaffinity/Cargo.toml @@ -12,7 +12,7 @@ tokio-postgres = { version = "0.7.0" } r2d2_postgres = " 0.18.0" r2d2 = "0.8" chrono = "0.4" -hyper = "0.14" +hyper = { version = "0.14", features = ["server"] } prometheus = { version = "0.11", features = ["process"] } lazy_static = "1" diff --git a/fuzzysearch-ingest-furaffinity/Dockerfile b/fuzzysearch-ingest-furaffinity/Dockerfile index c59e268..c72e5e1 100644 --- a/fuzzysearch-ingest-furaffinity/Dockerfile +++ b/fuzzysearch-ingest-furaffinity/Dockerfile @@ -1,5 +1,14 @@ -FROM rustlang/rust:nightly-slim +FROM rust:1-slim-buster AS builder +WORKDIR /src +ENV SQLX_OFFLINE=true +RUN apt-get update -y && apt-get install -y libssl-dev pkg-config python3 python3-pip COPY . . -RUN apt-get -y update && apt-get -y install pkg-config libssl-dev -RUN cargo install --root / --path . -CMD ["/bin/fa-watcher"] +RUN cargo install --root / --path ./fuzzysearch-ingest-furaffinity + +FROM rustlang/rust:nightly-slim +RUN apt-get update -y && \ + apt-get install -y openssl ca-certificates python3 python3-pip && \ + python3 -m pip --no-cache-dir install cfscrape && \ + rm -rf /var/lib/apt/lists/* +COPY --from=builder /bin/fuzzysearch-ingest-furaffinity /bin/fuzzysearch-ingest-furaffinity +CMD ["/bin/fuzzysearch-ingest-furaffinity"] diff --git a/fuzzysearch-ingest-weasyl/Cargo.toml b/fuzzysearch-ingest-weasyl/Cargo.toml index 512366d..027b36d 100644 --- a/fuzzysearch-ingest-weasyl/Cargo.toml +++ b/fuzzysearch-ingest-weasyl/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "weasyl-watcher" +name = "fuzzysearch-ingest-weasyl" version = "0.1.0" authors = ["Syfaro "] edition = "2018" diff --git a/fuzzysearch-ingest-weasyl/Dockerfile b/fuzzysearch-ingest-weasyl/Dockerfile index 082fce3..0d398bb 100644 --- a/fuzzysearch-ingest-weasyl/Dockerfile +++ b/fuzzysearch-ingest-weasyl/Dockerfile @@ -1,4 +1,11 @@ -FROM debian:buster-slim -RUN apt-get update -y && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/* -COPY ./weasyl-watcher /bin/weasyl-watcher -CMD ["/bin/weasyl-watcher"] +FROM rust:1-slim-buster AS builder +WORKDIR /src +ENV SQLX_OFFLINE=true +RUN apt-get update -y && apt-get install -y libssl-dev pkg-config +COPY . . +RUN cargo install --root / --path ./fuzzysearch-ingest-weasyl + +FROM rustlang/rust:nightly-slim +RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/* +COPY --from=builder /bin/fuzzysearch-ingest-weasyl /bin/fuzzysearch-ingest-weasyl +CMD ["/bin/fuzzysearch-ingest-weasyl"] diff --git a/fuzzysearch/Cargo.toml b/fuzzysearch/Cargo.toml index a624e6c..9fa6345 100644 --- a/fuzzysearch/Cargo.toml +++ b/fuzzysearch/Cargo.toml @@ -41,4 +41,4 @@ hamming = "0.1" bk-tree = "0.3" -fuzzysearch-common = { path = "../fuzzysearch-common" } +fuzzysearch-common = { path = "../fuzzysearch-common", features = ["video"] } diff --git a/fuzzysearch/Dockerfile b/fuzzysearch/Dockerfile index 253ff7e..ef73344 100644 --- a/fuzzysearch/Dockerfile +++ b/fuzzysearch/Dockerfile @@ -1,9 +1,9 @@ -FROM rust:1-slim AS builder +FROM rust:1-slim-buster AS builder WORKDIR /src ENV SQLX_OFFLINE=true -RUN apt-get update -y && apt-get install -y libssl-dev pkg-config +RUN apt-get update -y && apt-get install -y libssl-dev pkg-config libavcodec-dev libavformat-dev libavutil-dev libavdevice-dev clang llvm COPY . . -RUN cargo install --root / --path . +RUN cargo install --root / --path ./fuzzysearch FROM debian:buster-slim EXPOSE 8080 8081