experiments uploading of the tags to neo4j

main.go

@@ -3,8 +3,10 @@ package main
 import (
 	"context"
 	"e621_to_neo4j/e621/api"
-	"e621_to_neo4j/neo4j"
+	"e621_to_neo4j/e621/api/models"
+	"e621_to_neo4j/neo4jAPI"
 	"e621_to_neo4j/utils"
+	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	"log"
 	"time"
 )
@@ -15,7 +17,7 @@ func main() {
 		log.Println(err)
 	}
 
-	driver, err := neo4j.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
+	driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -29,7 +31,7 @@ func main() {
 		log.Fatal(err)
 	}
 
-	err = neo4j.CreateUserNode(ctx, driver, user)
+	err = neo4jAPI.CreateUserNode(ctx, driver, user)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -38,7 +40,15 @@ func main() {
 	if err != nil {
 		log.Fatal(err)
 	}
+
 	start := time.Now()
+	Upload2(favs, ctx, driver)
+	elapsed := time.Since(start)
+
+	log.Printf("This took %s", elapsed)
+}
+
+func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
 	for i, fav := range favs {
 
 		log.Printf("The e621 post with the id %d has %d general Tags, %d character Tags, %d copyright Tags, %d artist Tags.", fav.ID, len(fav.Tags.General), len(fav.Tags.Character), len(fav.Tags.Copyright), len(fav.Tags.Artist))
@@ -46,14 +56,14 @@ func main() {
 
 		for _, general := range fav.Tags.General {
 			log.Printf("TagType: General - Tag: %s", general)
-			err := neo4j.CreateTagNode(ctx, driver, general, "general")
+			err := neo4jAPI.CreateTagNode(ctx, driver, general, "general")
 			if err != nil {
 				log.Fatal(err)
 			}
 		}
 		for _, character := range fav.Tags.Character {
 			log.Printf("TagType: Character - Tag: %s", character)
-			err := neo4j.CreateTagNode(ctx, driver, character, "character")
+			err := neo4jAPI.CreateTagNode(ctx, driver, character, "character")
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -61,7 +71,7 @@ func main() {
 
 		for _, copyright := range fav.Tags.Copyright {
 			log.Printf("TagType: Copyright - Tag: %s", copyright)
-			err := neo4j.CreateTagNode(ctx, driver, copyright, "copyright")
+			err := neo4jAPI.CreateTagNode(ctx, driver, copyright, "copyright")
 			if err != nil {
 				log.Fatal(err)
 			}
@@ -69,12 +79,94 @@ func main() {
 
 		for _, artist := range fav.Tags.Artist {
 			log.Printf("TagType: Artist - Tag: %s", artist)
-			err := neo4j.CreateTagNode(ctx, driver, artist, "artist")
+			err := neo4jAPI.CreateTagNode(ctx, driver, artist, "artist")
 			if err != nil {
 				log.Fatal(err)
 			}
 		}
 	}
-	elapsed := time.Since(start)
+}
-	log.Printf("This took %s", elapsed)
+
+func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
+	uniqueGeneralTags := make(map[string]struct{})
+	uniqueCharacterTags := make(map[string]struct{})
+	uniqueCopyrightTags := make(map[string]struct{})
+	uniqueArtistTags := make(map[string]struct{})
+
+	allGeneralTags := make([]string, 0)
+	allCharacterTags := make([]string, 0)
+	allCopyrightTags := make([]string, 0)
+	allArtistTags := make([]string, 0)
+
+	for _, fav := range favs {
+
+		// Process General Tags
+		for _, general := range fav.Tags.General {
+			uniqueGeneralTags[general] = struct{}{}
+			allGeneralTags = append(allGeneralTags, general)
+		}
+
+		// Process Character Tags
+		for _, character := range fav.Tags.Character {
+			uniqueCharacterTags[character] = struct{}{}
+			allCharacterTags = append(allCharacterTags, character)
+		}
+
+		// Process Copyright Tags
+		for _, copyright := range fav.Tags.Copyright {
+			uniqueCopyrightTags[copyright] = struct{}{}
+			allCopyrightTags = append(allCopyrightTags, copyright)
+		}
+
+		// Process Artist Tags
+		for _, artist := range fav.Tags.Artist {
+			uniqueArtistTags[artist] = struct{}{}
+			allArtistTags = append(allArtistTags, artist)
+		}
+
+	}
+
+	log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
+	log.Printf("allGeneralTags length: %d", len(allGeneralTags))
+
+	log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
+	log.Printf("allCharacterTags length: %d", len(allCharacterTags))
+
+	log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
+	log.Printf("alleCopyrightTags length: %d", len(allCopyrightTags))
+
+	log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
+	log.Printf("allArtistTags length: %d", len(allArtistTags))
+
+	for uniqueGeneralTag := range uniqueGeneralTags {
+		log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+	for uniqueCharacterTag := range uniqueCharacterTags {
+		log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for uniqueCopyrightTag := range uniqueCopyrightTags {
+		log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
+	for uniqueArtistTag := range uniqueArtistTags {
+		log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
+		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
+		if err != nil {
+			log.Fatal(err)
+		}
+	}
+
 }