Added another method making tags unique

main.go

@@ -42,12 +42,13 @@ func main() {
 	}
 
 	start := time.Now()
-	Upload2(favs, ctx, driver)
+	Upload3(favs, ctx, driver)
 	elapsed := time.Since(start)
 
 	log.Printf("This took %s", elapsed)
 }
 
+// ~4min
 func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
 	for i, fav := range favs {
 
@@ -87,6 +88,7 @@ func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithCont
 	}
 }
 
+// ~30sec
 func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
 	uniqueGeneralTags := make(map[string]struct{})
 	uniqueCharacterTags := make(map[string]struct{})
@@ -170,3 +172,77 @@ func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithCon
 	}
 
 }
+
+// ~57s
+func Upload3(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
+	uniqueGeneralTags := make([]string, 0)
+	uniqueCharacterTags := make([]string, 0)
+	uniqueCopyrightTags := make([]string, 0)
+	uniqueArtistTags := make([]string, 0)
+
+	allGeneralTags := make([]string, 0)
+	allCharacterTags := make([]string, 0)
+	allCopyrightTags := make([]string, 0)
+	allArtistTags := make([]string, 0)
+
+	// add all tags together
+	for _, fav := range favs {
+		allGeneralTags = append(allGeneralTags, fav.Tags.General...)
+		allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
+		allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
+		allArtistTags = append(allArtistTags, fav.Tags.Artist...)
+	}
+
+	// Process General Tags
+	uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
+
+	// Process Character Tags
+	uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
+
+	// Process Copyright Tags
+	uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
+
+	// Process Artist Tags
+	uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
+
+	log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
+
+	log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
+
+	log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
+
+	log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
+
+	for _, uniqueGeneralTag := range uniqueGeneralTags {
+
+		log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	for _, uniqueCharacterTag := range uniqueCharacterTags {
+		log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for _, uniqueCopyrightTag := range uniqueCopyrightTags {
+		log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for _, uniqueArtistTag := range uniqueArtistTags {
+		log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+}

utils/list.go

@@ -0,0 +1,27 @@
+package utils
+
+// UniqueNonEmptyElementsOf returns a new slice containing unique non-empty elements from the input slice.
+// It removes duplicate elements and empty strings while preserving the order of appearance.
+func UniqueNonEmptyElementsOf(s []string) []string {
+	// Create a map to store unique elements
+	unique := make(map[string]bool)
+
+	// Create a new slice to store the unique non-empty elements
+	us := make([]string, 0, len(s))
+
+	for _, elem := range s {
+		// Skip empty strings
+		if len(elem) == 0 {
+			continue
+		}
+
+		// Check if the element is already in the unique map
+		if !unique[elem] {
+			// Add the element to the unique map and the new slice
+			unique[elem] = true
+			us = append(us, elem)
+		}
+	}
+
+	return us
+}