diff --git a/services/user.go b/services/user.go index 4b893c4..620f2e3 100644 --- a/services/user.go +++ b/services/user.go @@ -3,6 +3,7 @@ package services import ( "context" "e621_to_neo4j/e621" + "e621_to_neo4j/e621/models" "e621_to_neo4j/neo4jAPI" "e621_to_neo4j/utils" "github.com/neo4j/neo4j-go-driver/v5/neo4j" @@ -18,136 +19,223 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client return err } + if e621User.IsBanned { + log.Printf("User %s is banned from e621!", e621User.Name) + return nil + } + + log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID) + err = neo4jAPI.CreateUserNode(ctx, driver, e621User) if err != nil { log.Fatal(err) } + log.Printf("Getting favorites for user %s with id %d", e621User.Name, e621User.ID) + start := time.Now() userFavorites, err := e621Client.GetFavorites(e621User.Name) if err != nil { log.Fatal(err) } + log.Printf("User %s with id %d has %d favorites. Time took to scrape: %v", e621User.Name, e621User.ID, len(userFavorites), time.Since(start)) startUploadPosts := time.Now() + + // Uploads all Tags, Posts as Nodes to Neo4j + for i, post := range userFavorites { + start = time.Now() + err = uploadNodes(ctx, driver, post) + if err != nil { + return err + } + log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start)) + + } + + // Makes relationships between different nodes for i, post := range userFavorites { start := time.Now() - - uniqueGeneralTags := make([]string, 0) - uniqueCharacterTags := make([]string, 0) - uniqueCopyrightTags := make([]string, 0) - uniqueArtistTags := make([]string, 0) - - allGeneralTags := make([]string, 0) - allCharacterTags := make([]string, 0) - allCopyrightTags := make([]string, 0) - allArtistTags := make([]string, 0) - - allGeneralTags = append(allGeneralTags, post.Tags.General...) - allCharacterTags = append(allCharacterTags, post.Tags.Character...) - allCopyrightTags = append(allCopyrightTags, post.Tags.Character...) - allArtistTags = append(allArtistTags, post.Tags.Artist...) - - uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) - uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) - uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) - uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) - - // Uploads post to database - err := neo4jAPI.CreatePostNode(ctx, driver, post.ID) + err = uploadPostToUserRelationship(ctx, driver, post, e621User) if err != nil { + log.Fatal(err) return err } - // Uploads the source to the database - for _, source := range post.Sources { - err := neo4jAPI.CreateSourceNode(ctx, driver, source) - if err != nil { - return err - } - } - - for _, uniqueGeneralTag := range uniqueGeneralTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") - if err != nil { - return err - } - } - for _, uniqueCharacterTag := range uniqueCharacterTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") - if err != nil { - return err - } - } - - for _, uniqueCopyrightTag := range uniqueCopyrightTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") - if err != nil { - return err - } - } - - for _, uniqueArtistTag := range uniqueArtistTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") - if err != nil { - return err - } - } - log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start)) - } - - for _, post := range userFavorites { - id := post.ID - - err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID) + err = uploadSourceTagRelationship(ctx, driver, post) if err != nil { + log.Fatal(err) return err } - log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID) - - postID := post.ID - for _, source := range post.Sources { - err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source) - if err != nil { - return err - } - log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source) + err = uploadGeneralTagRelationship(ctx, driver, post) + if err != nil { + log.Fatal(err) + return err } - for _, generalTag := range post.Tags.General { - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag) - if err != nil { - log.Println(err) - } - log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag) - + err = uploadCharacterTagtRelationship(ctx, driver, post) + if err != nil { + log.Fatal(err) + return err } - for _, characterTag := range post.Tags.Character { - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag) - if err != nil { - log.Println(err) - } - log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag) + err = uploadCopyrightTagRelationship(ctx, driver, post) + if err != nil { + log.Fatal(err) + return err } - for _, copyrightTag := range post.Tags.Copyright { - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag) - if err != nil { - log.Println(err) - } - log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag) + err = uploadArtistTagRelationship(ctx, driver, post) + if err != nil { + log.Fatal(err) + return err } - for _, artistTag := range post.Tags.Artist { - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag) - if err != nil { - log.Println(err) - } - log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag) + log.Printf("Making relationship for user %s with id %d, %d for Post: %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start)) - } } - log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts)) + log.Printf("Uploading all posts for user %s took: %v", username, time.Since(startUploadPosts)) return nil } + +// uploadNodes uploads the post to the database and creates the nodes +func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { + uniqueGeneralTags := make([]string, 0) + uniqueCharacterTags := make([]string, 0) + uniqueCopyrightTags := make([]string, 0) + uniqueArtistTags := make([]string, 0) + + allGeneralTags := make([]string, 0) + allCharacterTags := make([]string, 0) + allCopyrightTags := make([]string, 0) + allArtistTags := make([]string, 0) + + allGeneralTags = append(allGeneralTags, post.Tags.General...) + allCharacterTags = append(allCharacterTags, post.Tags.Character...) + allCopyrightTags = append(allCopyrightTags, post.Tags.Character...) + allArtistTags = append(allArtistTags, post.Tags.Artist...) + + uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) + uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) + uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) + uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) + + // Uploads post to database + err := neo4jAPI.CreatePostNode(ctx, driver, post.ID) + if err != nil { + return err + } + + // Uploads the source to the database + for _, source := range post.Sources { + err := neo4jAPI.CreateSourceNode(ctx, driver, source) + if err != nil { + return err + } + } + + for _, uniqueGeneralTag := range uniqueGeneralTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") + if err != nil { + return err + } + } + for _, uniqueCharacterTag := range uniqueCharacterTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") + if err != nil { + return err + } + } + + for _, uniqueCopyrightTag := range uniqueCopyrightTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") + if err != nil { + return err + } + } + + for _, uniqueArtistTag := range uniqueArtistTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") + if err != nil { + return err + } + } + return nil +} + +// uploadPostToUserRelationship creates a relationship between the user and the post +func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post, e621User models.E621User) error { + err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID) + if err != nil { + return err + } + // log.Printf("Created UserToPostRelationship for user: %s to post: %d", e621User.Name, post.ID) + return nil +} + +// uploadSourceTagRelationship creates a relationship between the post and the source +func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { + for _, source := range post.Sources { + err := neo4jAPI.PostToSourceRelationship(ctx, driver, post.ID, source) + if err != nil { + return err + } + // log.Printf("Created PostToSourceRelationship for Post: %d to source: %s", post.ID, source) + + } + return nil +} + +// uploadGeneralTagRelationship creates a relationship between the post and the general tag +func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { + for _, generalTag := range post.Tags.General { + err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, generalTag) + if err != nil { + return err + } + // log.Printf("Created PostToTagRelationship for post: %d to general tag: %s", post.ID, generalTag) + + } + return nil +} + +// uploadCharacterTagtRelationship creates a relationship between the post and the character tag +func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { + for _, characterTag := range post.Tags.Character { + err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, characterTag) + if err != nil { + return err + } + // log.Printf("Created PostToTagRelationship for post: %d to character tag: %s", post.ID, characterTag) + + } + return nil +} + +// uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag +func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { + for _, copyrightTag := range post.Tags.Copyright { + err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, copyrightTag) + if err != nil { + return err + } + // log.Printf("Created PostToTagRelationship for post: %d to copyrigh tag: %s", post.ID, copyrightTag) + + } + return nil +} + +// uploadArtistTagRelationship creates a relationship between the post and the artist tag +func uploadArtistTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { + for _, artistTag := range post.Tags.Artist { + err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, artistTag) + if err != nil { + return err + } + // log.Printf("Created PostToTagRelationship for post: %d to artist tag: %s", post.ID, artistTag) + + } + return nil +} + +//11min für Selloo, simultan mit mutt_jake