Some updates for FurAffinity ingester.

This commit is contained in:
Syfaro 2020-12-07 17:57:13 -06:00
parent 3d7add5c1d
commit 348352f23e
5 changed files with 1048 additions and 74 deletions

View File

@ -15,18 +15,4 @@ steps:
- cargo build - cargo build
- cargo test - cargo test
- name: build-latest
image: plugins/docker
settings:
auto_tag: true
password:
from_secret: docker_password
registry: registry.huefox.com
repo: registry.huefox.com/fuzzysearch
username:
from_secret: docker_username
when:
branch:
- master
... ...

1077
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,7 +1,8 @@
[workspace] [workspace]
members = [ members = [
"fuzzysearch", "fuzzysearch",
"fuzzysearch-common" "fuzzysearch-common",
"fuzzysearch-ingest-furaffinity"
] ]
[profile.dev] [profile.dev]

View File

@ -15,7 +15,6 @@ chrono = "0.4"
hyper = "0.13" hyper = "0.13"
prometheus = { version = "0.11", features = ["process"] } prometheus = { version = "0.11", features = ["process"] }
lazy_static = "1" lazy_static = "1"
hex = "0.4"
[dependencies.furaffinity-rs] [dependencies.furaffinity-rs]
git = "https://github.com/Syfaro/furaffinity-rs" git = "https://github.com/Syfaro/furaffinity-rs"

View File

@ -73,7 +73,7 @@ async fn ids_to_check(client: &Client, max: i32) -> Vec<i32> {
async fn insert_submission( async fn insert_submission(
client: &Client, client: &Client,
sub: &furaffinity_rs::Submission, sub: &furaffinity_rs::Submission,
) -> Result<(), postgres::Error> { ) -> Result<(), tokio_postgres::Error> {
let artist_id = lookup_artist(&client, &sub.artist).await; let artist_id = lookup_artist(&client, &sub.artist).await;
let mut tag_ids = Vec::with_capacity(sub.tags.len()); let mut tag_ids = Vec::with_capacity(sub.tags.len());
for tag in &sub.tags { for tag in &sub.tags {
@ -100,7 +100,7 @@ async fn insert_submission(
Ok(()) Ok(())
} }
async fn insert_null_submission(client: &Client, id: i32) -> Result<u64, postgres::Error> { async fn insert_null_submission(client: &Client, id: i32) -> Result<u64, tokio_postgres::Error> {
client client
.execute("INSERT INTO SUBMISSION (id) VALUES ($1)", &[&id]) .execute("INSERT INTO SUBMISSION (id) VALUES ($1)", &[&id])
.await .await
@ -153,9 +153,6 @@ async fn main() {
std::env::var("FA_B").expect("missing fa cookie b"), std::env::var("FA_B").expect("missing fa cookie b"),
); );
let path = std::env::var("OUT_DIR").expect("missing output directory");
let path = std::path::Path::new(&path);
let user_agent = std::env::var("USER_AGENT").expect("missing user agent"); let user_agent = std::env::var("USER_AGENT").expect("missing user agent");
let fa = furaffinity_rs::FurAffinity::new(cookie_a, cookie_b, user_agent); let fa = furaffinity_rs::FurAffinity::new(cookie_a, cookie_b, user_agent);
@ -223,26 +220,6 @@ async fn main() {
timer.stop_and_record(); timer.stop_and_record();
if let Some(sha) = &sub.file_sha256 {
use tokio::io::AsyncWriteExt;
let file = sub.file.as_ref().unwrap();
let ext = sub.filename.split('.').last().unwrap();
let h = hex::encode(sha);
let p = path.join(&h[0..2]).join(&h[2..4]);
std::fs::create_dir_all(&p).expect("unable to create hash directory");
let name = format!("{}.{}", h, ext);
let name = std::path::Path::new(&name);
let name = p.join(name);
if !name.exists() {
let mut f = tokio::fs::File::create(&name).await.expect("unable to create submission file");
f.write_all(file).await.expect("unable to write file contents");
}
}
insert_submission(&client, &sub).await.unwrap(); insert_submission(&client, &sub).await.unwrap();
break 'attempt; break 'attempt;