Update package names and Dockerfiles.

This commit is contained in:
Syfaro 2021-02-21 01:26:29 -05:00
parent e57d932042
commit 5f5a93c85b
15 changed files with 84 additions and 95 deletions

75
Cargo.lock generated
View File

@ -509,27 +509,6 @@ dependencies = [
"dtoa",
]
[[package]]
name = "e621-watcher"
version = "0.1.0"
dependencies = [
"anyhow",
"hamming",
"hyper",
"image",
"img_hash",
"lazy_static",
"prometheus 0.11.0",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "ego-tree"
version = "0.6.2"
@ -807,6 +786,28 @@ dependencies = [
"tracing",
]
[[package]]
name = "fuzzysearch-ingest-e621"
version = "0.1.0"
dependencies = [
"anyhow",
"fuzzysearch-common",
"hamming",
"hyper",
"image",
"img_hash",
"lazy_static",
"prometheus 0.11.0",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "fuzzysearch-ingest-furaffinity"
version = "0.1.0"
@ -824,6 +825,22 @@ dependencies = [
"tokio-postgres",
]
[[package]]
name = "fuzzysearch-ingest-weasyl"
version = "0.1.0"
dependencies = [
"anyhow",
"fuzzysearch-common",
"image",
"img_hash",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
]
[[package]]
name = "fxhash"
version = "0.2.1"
@ -3362,22 +3379,6 @@ version = "0.2.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64"
[[package]]
name = "weasyl-watcher"
version = "0.1.0"
dependencies = [
"anyhow",
"fuzzysearch-common",
"image",
"img_hash",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
]
[[package]]
name = "web-sys"
version = "0.3.47"

View File

@ -4,6 +4,11 @@ version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"
[features]
default = []
video = ["ffmpeg-next", "tempfile"]
[dependencies]
anyhow = "1"
tracing = "0.1"
@ -13,5 +18,5 @@ serde = { version = "1", features = ["derive"] }
image = "0.23"
img_hash = "3"
ffmpeg-next = "4"
tempfile = "3"
ffmpeg-next = { version = "4", optional = true }
tempfile = { version = "3", optional = true }

View File

@ -1,4 +1,5 @@
pub mod types;
#[cfg(feature = "video")]
pub mod video;
/// Create an instance of img_hash with project defaults.

View File

@ -1 +0,0 @@
target/

View File

@ -1,25 +0,0 @@
---
kind: pipeline
type: docker
name: default
platform:
os: linux
arch: amd64
steps:
- name: build-latest
image: plugins/docker
settings:
auto_tag: true
password:
from_secret: docker_password
registry: registry.huefox.com
repo: registry.huefox.com/e621-watcher
username:
from_secret: docker_username
when:
branch:
- master
...

View File

@ -1,2 +0,0 @@
/target
.env

View File

@ -1,5 +1,5 @@
[package]
name = "e621-watcher"
name = "fuzzysearch-ingest-e621"
version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"
@ -27,3 +27,5 @@ anyhow = "1"
lazy_static = "1"
prometheus = { version = "0.11", features = ["process"] }
fuzzysearch-common = { path = "../fuzzysearch-common" }

View File

@ -1,14 +1,14 @@
FROM rust:1-slim AS builder
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
COPY . .
RUN cargo install --root / --path .
RUN cargo install --root / --path ./fuzzysearch-ingest-e621
FROM debian:buster-slim
EXPOSE 8080
ENV METRICS_HOST=0.0.0.0:8080
WORKDIR /app
RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/*
COPY --from=builder /bin/e621-watcher /bin/e621-watcher
CMD ["/bin/e621-watcher"]
COPY --from=builder /bin/fuzzysearch-ingest-e621 /bin/fuzzysearch-ingest-e621
CMD ["/bin/fuzzysearch-ingest-e621"]

View File

@ -272,7 +272,7 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result<Image
tracing::trace!(?result, "Calculated image SHA256");
let hasher = get_hasher();
let hasher = fuzzysearch_common::get_hasher();
let img = match image::load_from_memory(&bytes) {
Ok(img) => img,
Err(err) => {
@ -292,14 +292,6 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result<Image
Ok((Some(hash), None, Some(result)))
}
fn get_hasher() -> img_hash::Hasher<[u8; 8]> {
img_hash::HasherConfig::with_bytes_type::<[u8; 8]>()
.hash_alg(img_hash::HashAlg::Gradient)
.hash_size(8, 8)
.preproc_dct()
.to_hasher()
}
async fn provide_metrics(
_: hyper::Request<hyper::Body>,
) -> Result<hyper::Response<hyper::Body>, std::convert::Infallible> {

View File

@ -12,7 +12,7 @@ tokio-postgres = { version = "0.7.0" }
r2d2_postgres = " 0.18.0"
r2d2 = "0.8"
chrono = "0.4"
hyper = "0.14"
hyper = { version = "0.14", features = ["server"] }
prometheus = { version = "0.11", features = ["process"] }
lazy_static = "1"

View File

@ -1,5 +1,14 @@
FROM rustlang/rust:nightly-slim
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config python3 python3-pip
COPY . .
RUN apt-get -y update && apt-get -y install pkg-config libssl-dev
RUN cargo install --root / --path .
CMD ["/bin/fa-watcher"]
RUN cargo install --root / --path ./fuzzysearch-ingest-furaffinity
FROM rustlang/rust:nightly-slim
RUN apt-get update -y && \
apt-get install -y openssl ca-certificates python3 python3-pip && \
python3 -m pip --no-cache-dir install cfscrape && \
rm -rf /var/lib/apt/lists/*
COPY --from=builder /bin/fuzzysearch-ingest-furaffinity /bin/fuzzysearch-ingest-furaffinity
CMD ["/bin/fuzzysearch-ingest-furaffinity"]

View File

@ -1,5 +1,5 @@
[package]
name = "weasyl-watcher"
name = "fuzzysearch-ingest-weasyl"
version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"

View File

@ -1,4 +1,11 @@
FROM debian:buster-slim
RUN apt-get update -y && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/*
COPY ./weasyl-watcher /bin/weasyl-watcher
CMD ["/bin/weasyl-watcher"]
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
COPY . .
RUN cargo install --root / --path ./fuzzysearch-ingest-weasyl
FROM rustlang/rust:nightly-slim
RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/*
COPY --from=builder /bin/fuzzysearch-ingest-weasyl /bin/fuzzysearch-ingest-weasyl
CMD ["/bin/fuzzysearch-ingest-weasyl"]

View File

@ -41,4 +41,4 @@ hamming = "0.1"
bk-tree = "0.3"
fuzzysearch-common = { path = "../fuzzysearch-common" }
fuzzysearch-common = { path = "../fuzzysearch-common", features = ["video"] }

View File

@ -1,9 +1,9 @@
FROM rust:1-slim AS builder
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config libavcodec-dev libavformat-dev libavutil-dev libavdevice-dev clang llvm
COPY . .
RUN cargo install --root / --path .
RUN cargo install --root / --path ./fuzzysearch
FROM debian:buster-slim
EXPOSE 8080 8081