mirror of
https://github.com/Syfaro/fuzzysearch.git
synced 2024-11-10 17:02:38 +00:00
Some updates for FurAffinity ingester.
This commit is contained in:
parent
3d7add5c1d
commit
348352f23e
14
.drone.yml
14
.drone.yml
@ -15,18 +15,4 @@ steps:
|
|||||||
- cargo build
|
- cargo build
|
||||||
- cargo test
|
- cargo test
|
||||||
|
|
||||||
- name: build-latest
|
|
||||||
image: plugins/docker
|
|
||||||
settings:
|
|
||||||
auto_tag: true
|
|
||||||
password:
|
|
||||||
from_secret: docker_password
|
|
||||||
registry: registry.huefox.com
|
|
||||||
repo: registry.huefox.com/fuzzysearch
|
|
||||||
username:
|
|
||||||
from_secret: docker_username
|
|
||||||
when:
|
|
||||||
branch:
|
|
||||||
- master
|
|
||||||
|
|
||||||
...
|
...
|
||||||
|
1077
Cargo.lock
generated
1077
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
@ -1,7 +1,8 @@
|
|||||||
[workspace]
|
[workspace]
|
||||||
members = [
|
members = [
|
||||||
"fuzzysearch",
|
"fuzzysearch",
|
||||||
"fuzzysearch-common"
|
"fuzzysearch-common",
|
||||||
|
"fuzzysearch-ingest-furaffinity"
|
||||||
]
|
]
|
||||||
|
|
||||||
[profile.dev]
|
[profile.dev]
|
||||||
|
@ -15,7 +15,6 @@ chrono = "0.4"
|
|||||||
hyper = "0.13"
|
hyper = "0.13"
|
||||||
prometheus = { version = "0.11", features = ["process"] }
|
prometheus = { version = "0.11", features = ["process"] }
|
||||||
lazy_static = "1"
|
lazy_static = "1"
|
||||||
hex = "0.4"
|
|
||||||
|
|
||||||
[dependencies.furaffinity-rs]
|
[dependencies.furaffinity-rs]
|
||||||
git = "https://github.com/Syfaro/furaffinity-rs"
|
git = "https://github.com/Syfaro/furaffinity-rs"
|
||||||
|
@ -73,7 +73,7 @@ async fn ids_to_check(client: &Client, max: i32) -> Vec<i32> {
|
|||||||
async fn insert_submission(
|
async fn insert_submission(
|
||||||
client: &Client,
|
client: &Client,
|
||||||
sub: &furaffinity_rs::Submission,
|
sub: &furaffinity_rs::Submission,
|
||||||
) -> Result<(), postgres::Error> {
|
) -> Result<(), tokio_postgres::Error> {
|
||||||
let artist_id = lookup_artist(&client, &sub.artist).await;
|
let artist_id = lookup_artist(&client, &sub.artist).await;
|
||||||
let mut tag_ids = Vec::with_capacity(sub.tags.len());
|
let mut tag_ids = Vec::with_capacity(sub.tags.len());
|
||||||
for tag in &sub.tags {
|
for tag in &sub.tags {
|
||||||
@ -100,7 +100,7 @@ async fn insert_submission(
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn insert_null_submission(client: &Client, id: i32) -> Result<u64, postgres::Error> {
|
async fn insert_null_submission(client: &Client, id: i32) -> Result<u64, tokio_postgres::Error> {
|
||||||
client
|
client
|
||||||
.execute("INSERT INTO SUBMISSION (id) VALUES ($1)", &[&id])
|
.execute("INSERT INTO SUBMISSION (id) VALUES ($1)", &[&id])
|
||||||
.await
|
.await
|
||||||
@ -153,9 +153,6 @@ async fn main() {
|
|||||||
std::env::var("FA_B").expect("missing fa cookie b"),
|
std::env::var("FA_B").expect("missing fa cookie b"),
|
||||||
);
|
);
|
||||||
|
|
||||||
let path = std::env::var("OUT_DIR").expect("missing output directory");
|
|
||||||
let path = std::path::Path::new(&path);
|
|
||||||
|
|
||||||
let user_agent = std::env::var("USER_AGENT").expect("missing user agent");
|
let user_agent = std::env::var("USER_AGENT").expect("missing user agent");
|
||||||
|
|
||||||
let fa = furaffinity_rs::FurAffinity::new(cookie_a, cookie_b, user_agent);
|
let fa = furaffinity_rs::FurAffinity::new(cookie_a, cookie_b, user_agent);
|
||||||
@ -223,26 +220,6 @@ async fn main() {
|
|||||||
|
|
||||||
timer.stop_and_record();
|
timer.stop_and_record();
|
||||||
|
|
||||||
if let Some(sha) = &sub.file_sha256 {
|
|
||||||
use tokio::io::AsyncWriteExt;
|
|
||||||
|
|
||||||
let file = sub.file.as_ref().unwrap();
|
|
||||||
let ext = sub.filename.split('.').last().unwrap();
|
|
||||||
|
|
||||||
let h = hex::encode(sha);
|
|
||||||
let p = path.join(&h[0..2]).join(&h[2..4]);
|
|
||||||
std::fs::create_dir_all(&p).expect("unable to create hash directory");
|
|
||||||
|
|
||||||
let name = format!("{}.{}", h, ext);
|
|
||||||
let name = std::path::Path::new(&name);
|
|
||||||
let name = p.join(name);
|
|
||||||
|
|
||||||
if !name.exists() {
|
|
||||||
let mut f = tokio::fs::File::create(&name).await.expect("unable to create submission file");
|
|
||||||
f.write_all(file).await.expect("unable to write file contents");
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
insert_submission(&client, &sub).await.unwrap();
|
insert_submission(&client, &sub).await.unwrap();
|
||||||
|
|
||||||
break 'attempt;
|
break 'attempt;
|
||||||
|
Loading…
Reference in New Issue
Block a user