From 11dac75882687788d4083cb26aa1d7428971954e Mon Sep 17 00:00:00 2001 From: David Janowski Date: Mon, 22 May 2023 21:01:59 +0200 Subject: [PATCH] nwo able to upload the sources to the DB --- main.go | 114 ++++++++++++--------------------------------- neo4jAPI/source.go | 2 +- 2 files changed, 30 insertions(+), 86 deletions(-) diff --git a/main.go b/main.go index 74381ba..c238e6d 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,12 @@ func main() { } ctx := context.Background() - defer driver.Close(ctx) + defer func(driver neo4j.DriverWithContext, ctx context.Context) { + err := driver.Close(ctx) + if err != nil { + log.Fatal(err) + } + }(driver, ctx) e621Client := api.NewClient(config.E621APIKey, config.E621Username) user, err := e621Client.GetUserInfo("selloo") @@ -36,29 +41,30 @@ func main() { log.Fatal(err) } - favs, err := e621Client.GetFavorites(user.Name) + favorites, err := e621Client.GetFavorites(user.Name) if err != nil { log.Fatal(err) } start := time.Now() - uploadTags(favs, ctx, driver) + uploadTags(favorites, ctx, driver) elapsed := time.Since(start) log.Printf("upload of Tags took %d", elapsed) start = time.Now() - uploadPosts(favs, ctx, driver) + uploadPosts(favorites, ctx, driver) elapsed = time.Since(start) log.Printf("upload of Posts took %d", elapsed) start = time.Now() - //uploadSources(favs, ctx, driver) + uploadSources(favorites, ctx, driver) elapsed = time.Since(start) log.Printf("upload of Sources took %d", elapsed) } -func uploadPosts(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { - for _, fav := range favs { +func uploadPosts(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { + for _, fav := range posts { + log.Printf("e621 PostID: %d", fav.ID) err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID) if err != nil { return err @@ -69,7 +75,7 @@ func uploadPosts(favs []models.Post, ctx context.Context, driver neo4j.DriverWit } -func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { +func uploadTags(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { uniqueGeneralTags := make([]string, 0) uniqueCharacterTags := make([]string, 0) uniqueCopyrightTags := make([]string, 0) @@ -81,7 +87,7 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith allArtistTags := make([]string, 0) // add all tags together - for _, fav := range favs { + for _, fav := range posts { allGeneralTags = append(allGeneralTags, fav.Tags.General...) allCharacterTags = append(allCharacterTags, fav.Tags.Character...) allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...) @@ -102,8 +108,6 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) - log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags)) - log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) @@ -113,14 +117,14 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") if err != nil { - log.Fatal(err) + return err } } for _, uniqueCharacterTag := range uniqueCharacterTags { log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") if err != nil { - log.Fatal(err) + return err } } @@ -128,7 +132,7 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") if err != nil { - log.Fatal(err) + return err } } @@ -136,81 +140,21 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") if err != nil { - log.Fatal(err) + return err } } - + return nil } -func uploadSources(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { - uniqueGeneralTags := make([]string, 0) - uniqueCharacterTags := make([]string, 0) - uniqueCopyrightTags := make([]string, 0) - uniqueArtistTags := make([]string, 0) - - allGeneralTags := make([]string, 0) - allCharacterTags := make([]string, 0) - allCopyrightTags := make([]string, 0) - allArtistTags := make([]string, 0) - - // add all tags together - for _, fav := range favs { - allGeneralTags = append(allGeneralTags, fav.Tags.General...) - allCharacterTags = append(allCharacterTags, fav.Tags.Character...) - allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...) - allArtistTags = append(allArtistTags, fav.Tags.Artist...) - } - - // Process General PostTags - uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) - - // Process Character PostTags - uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) - - // Process Copyright PostTags - uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) - - // Process Artist PostTags - uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) - - log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) - - log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags)) - - log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) - - log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) - - for _, uniqueGeneralTag := range uniqueGeneralTags { - - log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") - if err != nil { - log.Fatal(err) +func uploadSources(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { + for _, fav := range posts { + for _, source := range fav.Sources { + log.Printf("e621 Source: %s", source) + err := neo4jAPI.CreateSourceNode(ctx, driver, source) + if err != nil { + return err + } } } - for _, uniqueCharacterTag := range uniqueCharacterTags { - log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") - if err != nil { - log.Fatal(err) - } - } - - for _, uniqueCopyrightTag := range uniqueCopyrightTags { - log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") - if err != nil { - log.Fatal(err) - } - } - - for _, uniqueArtistTag := range uniqueArtistTags { - log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") - if err != nil { - log.Fatal(err) - } - } - + return nil } diff --git a/neo4jAPI/source.go b/neo4jAPI/source.go index 3a2c3c6..b6a6cf3 100644 --- a/neo4jAPI/source.go +++ b/neo4jAPI/source.go @@ -11,7 +11,7 @@ func CreateSourceNode(ctx context.Context, driver neo4j.DriverWithContext, URL s RETURN u ` params := map[string]any{ - "URL": URL, + "url": URL, } _, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)