From 01fcb0019127a172eb1425acd726012cb1fd5341 Mon Sep 17 00:00:00 2001 From: David Janowski Date: Mon, 22 May 2023 16:00:49 +0200 Subject: [PATCH] experiments uploading of the tags to neo4j --- main.go | 110 +++++++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 101 insertions(+), 9 deletions(-) diff --git a/main.go b/main.go index 5608b66..34e999f 100644 --- a/main.go +++ b/main.go @@ -3,8 +3,10 @@ package main import ( "context" "e621_to_neo4j/e621/api" - "e621_to_neo4j/neo4j" + "e621_to_neo4j/e621/api/models" + "e621_to_neo4j/neo4jAPI" "e621_to_neo4j/utils" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" "log" "time" ) @@ -15,7 +17,7 @@ func main() { log.Println(err) } - driver, err := neo4j.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword) + driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword) if err != nil { log.Fatal(err) } @@ -29,7 +31,7 @@ func main() { log.Fatal(err) } - err = neo4j.CreateUserNode(ctx, driver, user) + err = neo4jAPI.CreateUserNode(ctx, driver, user) if err != nil { log.Fatal(err) } @@ -38,7 +40,15 @@ func main() { if err != nil { log.Fatal(err) } + start := time.Now() + Upload2(favs, ctx, driver) + elapsed := time.Since(start) + + log.Printf("This took %s", elapsed) +} + +func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { for i, fav := range favs { log.Printf("The e621 post with the id %d has %d general Tags, %d character Tags, %d copyright Tags, %d artist Tags.", fav.ID, len(fav.Tags.General), len(fav.Tags.Character), len(fav.Tags.Copyright), len(fav.Tags.Artist)) @@ -46,14 +56,14 @@ func main() { for _, general := range fav.Tags.General { log.Printf("TagType: General - Tag: %s", general) - err := neo4j.CreateTagNode(ctx, driver, general, "general") + err := neo4jAPI.CreateTagNode(ctx, driver, general, "general") if err != nil { log.Fatal(err) } } for _, character := range fav.Tags.Character { log.Printf("TagType: Character - Tag: %s", character) - err := neo4j.CreateTagNode(ctx, driver, character, "character") + err := neo4jAPI.CreateTagNode(ctx, driver, character, "character") if err != nil { log.Fatal(err) } @@ -61,7 +71,7 @@ func main() { for _, copyright := range fav.Tags.Copyright { log.Printf("TagType: Copyright - Tag: %s", copyright) - err := neo4j.CreateTagNode(ctx, driver, copyright, "copyright") + err := neo4jAPI.CreateTagNode(ctx, driver, copyright, "copyright") if err != nil { log.Fatal(err) } @@ -69,12 +79,94 @@ func main() { for _, artist := range fav.Tags.Artist { log.Printf("TagType: Artist - Tag: %s", artist) - err := neo4j.CreateTagNode(ctx, driver, artist, "artist") + err := neo4jAPI.CreateTagNode(ctx, driver, artist, "artist") if err != nil { log.Fatal(err) } } } - elapsed := time.Since(start) - log.Printf("This took %s", elapsed) +} + +func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { + uniqueGeneralTags := make(map[string]struct{}) + uniqueCharacterTags := make(map[string]struct{}) + uniqueCopyrightTags := make(map[string]struct{}) + uniqueArtistTags := make(map[string]struct{}) + + allGeneralTags := make([]string, 0) + allCharacterTags := make([]string, 0) + allCopyrightTags := make([]string, 0) + allArtistTags := make([]string, 0) + + for _, fav := range favs { + + // Process General Tags + for _, general := range fav.Tags.General { + uniqueGeneralTags[general] = struct{}{} + allGeneralTags = append(allGeneralTags, general) + } + + // Process Character Tags + for _, character := range fav.Tags.Character { + uniqueCharacterTags[character] = struct{}{} + allCharacterTags = append(allCharacterTags, character) + } + + // Process Copyright Tags + for _, copyright := range fav.Tags.Copyright { + uniqueCopyrightTags[copyright] = struct{}{} + allCopyrightTags = append(allCopyrightTags, copyright) + } + + // Process Artist Tags + for _, artist := range fav.Tags.Artist { + uniqueArtistTags[artist] = struct{}{} + allArtistTags = append(allArtistTags, artist) + } + + } + + log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) + log.Printf("allGeneralTags length: %d", len(allGeneralTags)) + + log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags)) + log.Printf("allCharacterTags length: %d", len(allCharacterTags)) + + log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) + log.Printf("alleCopyrightTags length: %d", len(allCopyrightTags)) + + log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) + log.Printf("allArtistTags length: %d", len(allArtistTags)) + + for uniqueGeneralTag := range uniqueGeneralTags { + log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") + if err != nil { + log.Fatal(err) + } + } + for uniqueCharacterTag := range uniqueCharacterTags { + log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") + if err != nil { + log.Fatal(err) + } + } + + for uniqueCopyrightTag := range uniqueCopyrightTags { + log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") + if err != nil { + log.Fatal(err) + } + } + + for uniqueArtistTag := range uniqueArtistTags { + log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") + if err != nil { + log.Fatal(err) + } + } + }