logging and cleanup
This commit is contained in:
David Janowski
parent
690201433d
commit
8113adba53
288
services/user.go
288
services/user.go
@ -3,6 +3,7 @@ package services
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"e621_to_neo4j/neo4jAPI"
|
||||
"e621_to_neo4j/utils"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
@ -18,136 +19,223 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
|
||||
return err
|
||||
}
|
||||
|
||||
if e621User.IsBanned {
|
||||
log.Printf("User %s is banned from e621!", e621User.Name)
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID)
|
||||
|
||||
err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Printf("Getting favorites for user %s with id %d", e621User.Name, e621User.ID)
|
||||
start := time.Now()
|
||||
userFavorites, err := e621Client.GetFavorites(e621User.Name)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("User %s with id %d has %d favorites. Time took to scrape: %v", e621User.Name, e621User.ID, len(userFavorites), time.Since(start))
|
||||
|
||||
startUploadPosts := time.Now()
|
||||
|
||||
// Uploads all Tags, Posts as Nodes to Neo4j
|
||||
for i, post := range userFavorites {
|
||||
start = time.Now()
|
||||
err = uploadNodes(ctx, driver, post)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))
|
||||
|
||||
}
|
||||
|
||||
// Makes relationships between different nodes
|
||||
for i, post := range userFavorites {
|
||||
start := time.Now()
|
||||
|
||||
uniqueGeneralTags := make([]string, 0)
|
||||
uniqueCharacterTags := make([]string, 0)
|
||||
uniqueCopyrightTags := make([]string, 0)
|
||||
uniqueArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags := make([]string, 0)
|
||||
allCharacterTags := make([]string, 0)
|
||||
allCopyrightTags := make([]string, 0)
|
||||
allArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags = append(allGeneralTags, post.Tags.General...)
|
||||
allCharacterTags = append(allCharacterTags, post.Tags.Character...)
|
||||
allCopyrightTags = append(allCopyrightTags, post.Tags.Character...)
|
||||
allArtistTags = append(allArtistTags, post.Tags.Artist...)
|
||||
|
||||
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
|
||||
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
|
||||
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
|
||||
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
|
||||
|
||||
// Uploads post to database
|
||||
err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
|
||||
err = uploadPostToUserRelationship(ctx, driver, post, e621User)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
|
||||
// Uploads the source to the database
|
||||
for _, source := range post.Sources {
|
||||
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueGeneralTag := range uniqueGeneralTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, uniqueCharacterTag := range uniqueCharacterTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueArtistTag := range uniqueArtistTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start))
|
||||
}
|
||||
|
||||
for _, post := range userFavorites {
|
||||
id := post.ID
|
||||
|
||||
err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
|
||||
err = uploadSourceTagRelationship(ctx, driver, post)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID)
|
||||
|
||||
postID := post.ID
|
||||
for _, source := range post.Sources {
|
||||
err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source)
|
||||
|
||||
err = uploadGeneralTagRelationship(ctx, driver, post)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
|
||||
for _, generalTag := range post.Tags.General {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag)
|
||||
|
||||
err = uploadCharacterTagtRelationship(ctx, driver, post)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
for _, characterTag := range post.Tags.Character {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag)
|
||||
|
||||
err = uploadCopyrightTagRelationship(ctx, driver, post)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
for _, copyrightTag := range post.Tags.Copyright {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag)
|
||||
|
||||
err = uploadArtistTagRelationship(ctx, driver, post)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
for _, artistTag := range post.Tags.Artist {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag)
|
||||
log.Printf("Making relationship for user %s with id %d, %d for Post: %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts))
|
||||
log.Printf("Uploading all posts for user %s took: %v", username, time.Since(startUploadPosts))
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadNodes uploads the post to the database and creates the nodes
|
||||
func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
|
||||
uniqueGeneralTags := make([]string, 0)
|
||||
uniqueCharacterTags := make([]string, 0)
|
||||
uniqueCopyrightTags := make([]string, 0)
|
||||
uniqueArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags := make([]string, 0)
|
||||
allCharacterTags := make([]string, 0)
|
||||
allCopyrightTags := make([]string, 0)
|
||||
allArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags = append(allGeneralTags, post.Tags.General...)
|
||||
allCharacterTags = append(allCharacterTags, post.Tags.Character...)
|
||||
allCopyrightTags = append(allCopyrightTags, post.Tags.Character...)
|
||||
allArtistTags = append(allArtistTags, post.Tags.Artist...)
|
||||
|
||||
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
|
||||
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
|
||||
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
|
||||
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
|
||||
|
||||
// Uploads post to database
|
||||
err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Uploads the source to the database
|
||||
for _, source := range post.Sources {
|
||||
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueGeneralTag := range uniqueGeneralTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, uniqueCharacterTag := range uniqueCharacterTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueArtistTag := range uniqueArtistTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadPostToUserRelationship creates a relationship between the user and the post
|
||||
func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post, e621User models.E621User) error {
|
||||
err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// log.Printf("Created UserToPostRelationship for user: %s to post: %d", e621User.Name, post.ID)
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadSourceTagRelationship creates a relationship between the post and the source
|
||||
func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
|
||||
for _, source := range post.Sources {
|
||||
err := neo4jAPI.PostToSourceRelationship(ctx, driver, post.ID, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// log.Printf("Created PostToSourceRelationship for Post: %d to source: %s", post.ID, source)
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadGeneralTagRelationship creates a relationship between the post and the general tag
|
||||
func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
|
||||
for _, generalTag := range post.Tags.General {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, generalTag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// log.Printf("Created PostToTagRelationship for post: %d to general tag: %s", post.ID, generalTag)
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadCharacterTagtRelationship creates a relationship between the post and the character tag
|
||||
func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
|
||||
for _, characterTag := range post.Tags.Character {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, characterTag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// log.Printf("Created PostToTagRelationship for post: %d to character tag: %s", post.ID, characterTag)
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag
|
||||
func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
|
||||
for _, copyrightTag := range post.Tags.Copyright {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, copyrightTag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// log.Printf("Created PostToTagRelationship for post: %d to copyrigh tag: %s", post.ID, copyrightTag)
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// uploadArtistTagRelationship creates a relationship between the post and the artist tag
|
||||
func uploadArtistTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
|
||||
for _, artistTag := range post.Tags.Artist {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, artistTag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// log.Printf("Created PostToTagRelationship for post: %d to artist tag: %s", post.ID, artistTag)
|
||||
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
//11min für Selloo, simultan mit mutt_jake
|
||||
|
||||
Reference in New Issue
Block a user