anthrove/fuzzysearch
4
0
Fork 0
mirror of https://github.com/Syfaro/fuzzysearch.git synced 2024-11-05 06:23:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Update package names and Dockerfiles.

Browse Source
This commit is contained in:
Syfaro 2021-02-21 01:26:29 -05:00
parent e57d932042
commit 5f5a93c85b
15 changed files with 84 additions and 95 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

75
Cargo.lock generated
View File

@ -509,27 +509,6 @@ dependencies = [
"dtoa",
]
[[package]]
name = "e621-watcher"
version = "0.1.0"
dependencies = [
"anyhow",
"hamming",
"hyper",
"image",
"img_hash",
"lazy_static",
"prometheus 0.11.0",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "ego-tree"
version = "0.6.2"
@ -807,6 +786,28 @@ dependencies = [
"tracing",
]
[[package]]
name = "fuzzysearch-ingest-e621"
version = "0.1.0"
dependencies = [
"anyhow",
"fuzzysearch-common",
"hamming",
"hyper",
"image",
"img_hash",
"lazy_static",
"prometheus 0.11.0",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
"tracing",
"tracing-subscriber",
]
[[package]]
name = "fuzzysearch-ingest-furaffinity"
version = "0.1.0"
@ -824,6 +825,22 @@ dependencies = [
"tokio-postgres",
]
[[package]]
name = "fuzzysearch-ingest-weasyl"
version = "0.1.0"
dependencies = [
"anyhow",
"fuzzysearch-common",
"image",
"img_hash",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
]
[[package]]
name = "fxhash"
version = "0.2.1"
@ -3362,22 +3379,6 @@ version = "0.2.70"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd4945e4943ae02d15c13962b38a5b1e81eadd4b71214eee75af64a4d6a4fd64"
[[package]]
name = "weasyl-watcher"
version = "0.1.0"
dependencies = [
"anyhow",
"fuzzysearch-common",
"image",
"img_hash",
"reqwest",
"serde",
"serde_json",
"sha2",
"sqlx",
"tokio",
]
[[package]]
name = "web-sys"
version = "0.3.47"

9
fuzzysearch-common/Cargo.toml
View File

@ -4,6 +4,11 @@ version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"
[features]
default = []
video = ["ffmpeg-next", "tempfile"]
[dependencies]
anyhow = "1"
tracing = "0.1"
@ -13,5 +18,5 @@ serde = { version = "1", features = ["derive"] }
image = "0.23"
img_hash = "3"
ffmpeg-next = "4"
tempfile = "3"
ffmpeg-next = { version = "4", optional = true }
tempfile = { version = "3", optional = true }

1
fuzzysearch-common/src/lib.rs
View File

@ -1,4 +1,5 @@
pub mod types;
#[cfg(feature = "video")]
pub mod video;
/// Create an instance of img_hash with project defaults.

1
fuzzysearch-ingest-e621/.dockerignore
View File

@ -1 +0,0 @@
target/

25
fuzzysearch-ingest-e621/.drone.yml
View File

@ -1,25 +0,0 @@
---
kind: pipeline
type: docker
name: default
platform:
os: linux
arch: amd64
steps:
- name: build-latest
image: plugins/docker
settings:
auto_tag: true
password:
from_secret: docker_password
registry: registry.huefox.com
repo: registry.huefox.com/e621-watcher
username:
from_secret: docker_username
when:
branch:
- master
...

2
fuzzysearch-ingest-e621/.gitignore vendored
View File

@ -1,2 +0,0 @@
/target
.env

4
fuzzysearch-ingest-e621/Cargo.toml
View File

@ -1,5 +1,5 @@
[package]
name = "e621-watcher"
name = "fuzzysearch-ingest-e621"
version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"
@ -27,3 +27,5 @@ anyhow = "1"
lazy_static = "1"
prometheus = { version = "0.11", features = ["process"] }
fuzzysearch-common = { path = "../fuzzysearch-common" }

8
fuzzysearch-ingest-e621/Dockerfile
View File

@ -1,14 +1,14 @@
FROM rust:1-slim AS builder
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
COPY . .
RUN cargo install --root / --path .
RUN cargo install --root / --path ./fuzzysearch-ingest-e621
FROM debian:buster-slim
EXPOSE 8080
ENV METRICS_HOST=0.0.0.0:8080
WORKDIR /app
RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/*
COPY --from=builder /bin/e621-watcher /bin/e621-watcher
CMD ["/bin/e621-watcher"]
COPY --from=builder /bin/fuzzysearch-ingest-e621 /bin/fuzzysearch-ingest-e621
CMD ["/bin/fuzzysearch-ingest-e621"]

10
fuzzysearch-ingest-e621/src/main.rs
View File

@ -272,7 +272,7 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result<Image
tracing::trace!(?result, "Calculated image SHA256");
let hasher = get_hasher();
let hasher = fuzzysearch_common::get_hasher();
let img = match image::load_from_memory(&bytes) {
Ok(img) => img,
Err(err) => {
@ -292,14 +292,6 @@ async fn load_image(client: &reqwest::Client, url: &str) -> anyhow::Result<Image
Ok((Some(hash), None, Some(result)))
}
fn get_hasher() -> img_hash::Hasher<[u8; 8]> {
img_hash::HasherConfig::with_bytes_type::<[u8; 8]>()
.hash_alg(img_hash::HashAlg::Gradient)
.hash_size(8, 8)
.preproc_dct()
.to_hasher()
}
async fn provide_metrics(
_: hyper::Request<hyper::Body>,
) -> Result<hyper::Response<hyper::Body>, std::convert::Infallible> {

2
fuzzysearch-ingest-furaffinity/Cargo.toml
View File

@ -12,7 +12,7 @@ tokio-postgres = { version = "0.7.0" }
r2d2_postgres = " 0.18.0"
r2d2 = "0.8"
chrono = "0.4"
hyper = "0.14"
hyper = { version = "0.14", features = ["server"] }
prometheus = { version = "0.11", features = ["process"] }
lazy_static = "1"

17
fuzzysearch-ingest-furaffinity/Dockerfile
View File

@ -1,5 +1,14 @@
FROM rustlang/rust:nightly-slim
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config python3 python3-pip
COPY . .
RUN apt-get -y update && apt-get -y install pkg-config libssl-dev
RUN cargo install --root / --path .
CMD ["/bin/fa-watcher"]
RUN cargo install --root / --path ./fuzzysearch-ingest-furaffinity
FROM rustlang/rust:nightly-slim
RUN apt-get update -y && \
apt-get install -y openssl ca-certificates python3 python3-pip && \
python3 -m pip --no-cache-dir install cfscrape && \
rm -rf /var/lib/apt/lists/*
COPY --from=builder /bin/fuzzysearch-ingest-furaffinity /bin/fuzzysearch-ingest-furaffinity
CMD ["/bin/fuzzysearch-ingest-furaffinity"]

2
fuzzysearch-ingest-weasyl/Cargo.toml
View File

@ -1,5 +1,5 @@
[package]
name = "weasyl-watcher"
name = "fuzzysearch-ingest-weasyl"
version = "0.1.0"
authors = ["Syfaro <syfaro@huefox.com>"]
edition = "2018"

15
fuzzysearch-ingest-weasyl/Dockerfile
View File

@ -1,4 +1,11 @@
FROM debian:buster-slim
RUN apt-get update -y && apt-get install -y libssl-dev ca-certificates && rm -rf /var/lib/apt/lists/*
COPY ./weasyl-watcher /bin/weasyl-watcher
CMD ["/bin/weasyl-watcher"]
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
COPY . .
RUN cargo install --root / --path ./fuzzysearch-ingest-weasyl
FROM rustlang/rust:nightly-slim
RUN apt-get update -y && apt-get install -y openssl ca-certificates && rm -rf /var/lib/apt/lists/*
COPY --from=builder /bin/fuzzysearch-ingest-weasyl /bin/fuzzysearch-ingest-weasyl
CMD ["/bin/fuzzysearch-ingest-weasyl"]

2
fuzzysearch/Cargo.toml
View File

@ -41,4 +41,4 @@ hamming = "0.1"
bk-tree = "0.3"
fuzzysearch-common = { path = "../fuzzysearch-common" }
fuzzysearch-common = { path = "../fuzzysearch-common", features = ["video"] }

6
fuzzysearch/Dockerfile
View File

@ -1,9 +1,9 @@
FROM rust:1-slim AS builder
FROM rust:1-slim-buster AS builder
WORKDIR /src
ENV SQLX_OFFLINE=true
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config
RUN apt-get update -y && apt-get install -y libssl-dev pkg-config libavcodec-dev libavformat-dev libavutil-dev libavdevice-dev clang llvm
COPY . .
RUN cargo install --root / --path .
RUN cargo install --root / --path ./fuzzysearch
FROM debian:buster-slim
EXPOSE 8080 8081