diff --git a/main.go b/main.go index 34e999f..3dc47c3 100644 --- a/main.go +++ b/main.go @@ -42,12 +42,13 @@ func main() { } start := time.Now() - Upload2(favs, ctx, driver) + Upload3(favs, ctx, driver) elapsed := time.Since(start) log.Printf("This took %s", elapsed) } +// ~4min func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { for i, fav := range favs { @@ -87,6 +88,7 @@ func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithCont } } +// ~30sec func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { uniqueGeneralTags := make(map[string]struct{}) uniqueCharacterTags := make(map[string]struct{}) @@ -170,3 +172,77 @@ func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithCon } } + +// ~57s +func Upload3(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { + uniqueGeneralTags := make([]string, 0) + uniqueCharacterTags := make([]string, 0) + uniqueCopyrightTags := make([]string, 0) + uniqueArtistTags := make([]string, 0) + + allGeneralTags := make([]string, 0) + allCharacterTags := make([]string, 0) + allCopyrightTags := make([]string, 0) + allArtistTags := make([]string, 0) + + // add all tags together + for _, fav := range favs { + allGeneralTags = append(allGeneralTags, fav.Tags.General...) + allCharacterTags = append(allCharacterTags, fav.Tags.Character...) + allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...) + allArtistTags = append(allArtistTags, fav.Tags.Artist...) + } + + // Process General Tags + uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) + + // Process Character Tags + uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) + + // Process Copyright Tags + uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) + + // Process Artist Tags + uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) + + log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) + + log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags)) + + log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) + + log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) + + for _, uniqueGeneralTag := range uniqueGeneralTags { + + log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") + if err != nil { + log.Fatal(err) + } + } + for _, uniqueCharacterTag := range uniqueCharacterTags { + log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") + if err != nil { + log.Fatal(err) + } + } + + for _, uniqueCopyrightTag := range uniqueCopyrightTags { + log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") + if err != nil { + log.Fatal(err) + } + } + + for _, uniqueArtistTag := range uniqueArtistTags { + log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") + if err != nil { + log.Fatal(err) + } + } + +} diff --git a/utils/list.go b/utils/list.go new file mode 100644 index 0000000..a4682f4 --- /dev/null +++ b/utils/list.go @@ -0,0 +1,27 @@ +package utils + +// UniqueNonEmptyElementsOf returns a new slice containing unique non-empty elements from the input slice. +// It removes duplicate elements and empty strings while preserving the order of appearance. +func UniqueNonEmptyElementsOf(s []string) []string { + // Create a map to store unique elements + unique := make(map[string]bool) + + // Create a new slice to store the unique non-empty elements + us := make([]string, 0, len(s)) + + for _, elem := range s { + // Skip empty strings + if len(elem) == 0 { + continue + } + + // Check if the element is already in the unique map + if !unique[elem] { + // Add the element to the unique map and the new slice + unique[elem] = true + us = append(us, elem) + } + } + + return us +}