package services import ( "context" "e621_to_neo4j/e621" "e621_to_neo4j/neo4jAPI" "e621_to_neo4j/utils" "github.com/neo4j/neo4j-go-driver/v5/neo4j" "log" "time" ) func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error { var err error e621User, err := e621Client.GetUserInfo(username) if err != nil { return err } err = neo4jAPI.CreateUserNode(ctx, driver, e621User) if err != nil { log.Fatal(err) } userFavorites, err := e621Client.GetFavorites(e621User.Name) if err != nil { log.Fatal(err) } startUploadPosts := time.Now() for i, post := range userFavorites { start := time.Now() uniqueGeneralTags := make([]string, 0) uniqueCharacterTags := make([]string, 0) uniqueCopyrightTags := make([]string, 0) uniqueArtistTags := make([]string, 0) allGeneralTags := make([]string, 0) allCharacterTags := make([]string, 0) allCopyrightTags := make([]string, 0) allArtistTags := make([]string, 0) allGeneralTags = append(allGeneralTags, post.Tags.General...) allCharacterTags = append(allCharacterTags, post.Tags.Character...) allCopyrightTags = append(allCopyrightTags, post.Tags.Character...) allArtistTags = append(allArtistTags, post.Tags.Artist...) uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) // Uploads post to database err := neo4jAPI.CreatePostNode(ctx, driver, post.ID) if err != nil { return err } // Uploads the source to the database for _, source := range post.Sources { err := neo4jAPI.CreateSourceNode(ctx, driver, source) if err != nil { return err } } for _, uniqueGeneralTag := range uniqueGeneralTags { err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") if err != nil { return err } } for _, uniqueCharacterTag := range uniqueCharacterTags { err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") if err != nil { return err } } for _, uniqueCopyrightTag := range uniqueCopyrightTags { err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") if err != nil { return err } } for _, uniqueArtistTag := range uniqueArtistTags { err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") if err != nil { return err } } log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start)) } for _, post := range userFavorites { id := post.ID err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID) if err != nil { return err } log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID) postID := post.ID for _, source := range post.Sources { err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source) if err != nil { return err } log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source) } for _, generalTag := range post.Tags.General { err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag) if err != nil { log.Println(err) } log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag) } for _, characterTag := range post.Tags.Character { err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag) if err != nil { log.Println(err) } log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag) } for _, copyrightTag := range post.Tags.Copyright { err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag) if err != nil { log.Println(err) } log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag) } for _, artistTag := range post.Tags.Artist { err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag) if err != nil { log.Println(err) } log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag) } } log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts)) return nil }