package main import ( "context" "e621_to_neo4j/e621/api" "e621_to_neo4j/e621/api/models" "e621_to_neo4j/neo4jAPI" "e621_to_neo4j/utils" "github.com/neo4j/neo4j-go-driver/v5/neo4j" "log" "time" ) func main() { config, err := utils.LoadConfig() if err != nil { log.Println(err) } driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword) if err != nil { log.Fatal(err) } ctx := context.Background() defer driver.Close(ctx) e621Client := api.NewClient(config.E621APIKey, config.E621Username) user, err := e621Client.GetUserInfo("selloo") if err != nil { log.Fatal(err) } err = neo4jAPI.CreateUserNode(ctx, driver, user) if err != nil { log.Fatal(err) } favs, err := e621Client.GetFavorites(user.Name) if err != nil { log.Fatal(err) } start := time.Now() Upload3(favs, ctx, driver) elapsed := time.Since(start) log.Printf("This took %s", elapsed) } // ~4min func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { for i, fav := range favs { log.Printf("The e621 post with the id %d has %d general Tags, %d character Tags, %d copyright Tags, %d artist Tags.", fav.ID, len(fav.Tags.General), len(fav.Tags.Character), len(fav.Tags.Copyright), len(fav.Tags.Artist)) log.Printf("Uploaded Posts: %d", i) for _, general := range fav.Tags.General { log.Printf("TagType: General - Tag: %s", general) err := neo4jAPI.CreateTagNode(ctx, driver, general, "general") if err != nil { log.Fatal(err) } } for _, character := range fav.Tags.Character { log.Printf("TagType: Character - Tag: %s", character) err := neo4jAPI.CreateTagNode(ctx, driver, character, "character") if err != nil { log.Fatal(err) } } for _, copyright := range fav.Tags.Copyright { log.Printf("TagType: Copyright - Tag: %s", copyright) err := neo4jAPI.CreateTagNode(ctx, driver, copyright, "copyright") if err != nil { log.Fatal(err) } } for _, artist := range fav.Tags.Artist { log.Printf("TagType: Artist - Tag: %s", artist) err := neo4jAPI.CreateTagNode(ctx, driver, artist, "artist") if err != nil { log.Fatal(err) } } } } // ~30sec func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { uniqueGeneralTags := make(map[string]struct{}) uniqueCharacterTags := make(map[string]struct{}) uniqueCopyrightTags := make(map[string]struct{}) uniqueArtistTags := make(map[string]struct{}) allGeneralTags := make([]string, 0) allCharacterTags := make([]string, 0) allCopyrightTags := make([]string, 0) allArtistTags := make([]string, 0) for _, fav := range favs { // Process General Tags for _, general := range fav.Tags.General { uniqueGeneralTags[general] = struct{}{} allGeneralTags = append(allGeneralTags, general) } // Process Character Tags for _, character := range fav.Tags.Character { uniqueCharacterTags[character] = struct{}{} allCharacterTags = append(allCharacterTags, character) } // Process Copyright Tags for _, copyright := range fav.Tags.Copyright { uniqueCopyrightTags[copyright] = struct{}{} allCopyrightTags = append(allCopyrightTags, copyright) } // Process Artist Tags for _, artist := range fav.Tags.Artist { uniqueArtistTags[artist] = struct{}{} allArtistTags = append(allArtistTags, artist) } } log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) log.Printf("allGeneralTags length: %d", len(allGeneralTags)) log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags)) log.Printf("allCharacterTags length: %d", len(allCharacterTags)) log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) log.Printf("alleCopyrightTags length: %d", len(allCopyrightTags)) log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) log.Printf("allArtistTags length: %d", len(allArtistTags)) for uniqueGeneralTag := range uniqueGeneralTags { log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") if err != nil { log.Fatal(err) } } for uniqueCharacterTag := range uniqueCharacterTags { log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") if err != nil { log.Fatal(err) } } for uniqueCopyrightTag := range uniqueCopyrightTags { log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") if err != nil { log.Fatal(err) } } for uniqueArtistTag := range uniqueArtistTags { log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") if err != nil { log.Fatal(err) } } } // ~57s func Upload3(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { uniqueGeneralTags := make([]string, 0) uniqueCharacterTags := make([]string, 0) uniqueCopyrightTags := make([]string, 0) uniqueArtistTags := make([]string, 0) allGeneralTags := make([]string, 0) allCharacterTags := make([]string, 0) allCopyrightTags := make([]string, 0) allArtistTags := make([]string, 0) // add all tags together for _, fav := range favs { allGeneralTags = append(allGeneralTags, fav.Tags.General...) allCharacterTags = append(allCharacterTags, fav.Tags.Character...) allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...) allArtistTags = append(allArtistTags, fav.Tags.Artist...) } // Process General Tags uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) // Process Character Tags uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) // Process Copyright Tags uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) // Process Artist Tags uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags)) log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) for _, uniqueGeneralTag := range uniqueGeneralTags { log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") if err != nil { log.Fatal(err) } } for _, uniqueCharacterTag := range uniqueCharacterTags { log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") if err != nil { log.Fatal(err) } } for _, uniqueCopyrightTag := range uniqueCopyrightTags { log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") if err != nil { log.Fatal(err) } } for _, uniqueArtistTag := range uniqueArtistTags { log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") if err != nil { log.Fatal(err) } } }