From 10c0247f3c186201049ced0059c2dd23cb0f6159 Mon Sep 17 00:00:00 2001 From: David Janowski Date: Wed, 24 May 2023 16:05:27 +0200 Subject: [PATCH] added user api implementation --- api/user.go | 36 ++++++ docker-compose.yml | 7 +- e621/{api => }/client.go | 2 +- e621/{api => }/favorite.go | 4 +- e621/{api => }/models/post.go | 8 +- e621/{api => }/models/user.go | 0 e621/{api => }/users.go | 4 +- main.go | 228 +++------------------------------- neo4jAPI/tag.go | 25 ++++ neo4jAPI/user.go | 2 +- services/user.go | 153 +++++++++++++++++++++++ 11 files changed, 247 insertions(+), 222 deletions(-) create mode 100644 api/user.go rename e621/{api => }/client.go (96%) rename e621/{api => }/favorite.go (96%) rename e621/{api => }/models/post.go (94%) rename e621/{api => }/models/user.go (100%) rename e621/{api => }/users.go (95%) create mode 100644 services/user.go diff --git a/api/user.go b/api/user.go new file mode 100644 index 0000000..b650c58 --- /dev/null +++ b/api/user.go @@ -0,0 +1,36 @@ +package api + +import ( + "context" + "e621_to_neo4j/e621" + "e621_to_neo4j/services" + "fmt" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + "net/http" +) + +// UserHandler is the handler for the user API +func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) { + return func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + fmt.Fprintf(w, "Only POST requests are allowed") + return + } + + username := r.FormValue("username") + if username == "" { + w.WriteHeader(http.StatusBadRequest) + fmt.Fprintf(w, "Username is required") + return + } + + // Perform further processing with the username + services.ScrapeUser(ctx, driver, *e621Client, username) + + // Send a response + w.WriteHeader(http.StatusOK) + fmt.Fprintf(w, "Username %s processed successfully", username) + + } +} diff --git a/docker-compose.yml b/docker-compose.yml index 891cf31..9724ca8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,4 +5,9 @@ services: ports: - '7474:7474' - '7473:7473' - - '7687:7687' \ No newline at end of file + - '7687:7687' + volumes: + - 'neo4j_data:/bitnami' +volumes: + neo4j_data: + driver: local \ No newline at end of file diff --git a/e621/api/client.go b/e621/client.go similarity index 96% rename from e621/api/client.go rename to e621/client.go index c95e7b4..31bed82 100644 --- a/e621/api/client.go +++ b/e621/client.go @@ -1,4 +1,4 @@ -package api +package e621 import ( "net/http" diff --git a/e621/api/favorite.go b/e621/favorite.go similarity index 96% rename from e621/api/favorite.go rename to e621/favorite.go index 284ceb0..607d002 100644 --- a/e621/api/favorite.go +++ b/e621/favorite.go @@ -1,7 +1,7 @@ -package api +package e621 import ( - "e621_to_neo4j/e621/api/models" + "e621_to_neo4j/e621/models" "fmt" "io" "log" diff --git a/e621/api/models/post.go b/e621/models/post.go similarity index 94% rename from e621/api/models/post.go rename to e621/models/post.go index fe153e2..0f16df1 100644 --- a/e621/api/models/post.go +++ b/e621/models/post.go @@ -2,17 +2,17 @@ package models import "encoding/json" -func UnmarshalE621Post(data []byte) (E621Post, error) { - var r E621Post +func UnmarshalE621Post(data []byte) (APIe621Post, error) { + var r APIe621Post err := json.Unmarshal(data, &r) return r, err } -func (r *E621Post) Marshal() ([]byte, error) { +func (r *APIe621Post) Marshal() ([]byte, error) { return json.Marshal(r) } -type E621Post struct { +type APIe621Post struct { Posts []Post `json:"posts"` } diff --git a/e621/api/models/user.go b/e621/models/user.go similarity index 100% rename from e621/api/models/user.go rename to e621/models/user.go diff --git a/e621/api/users.go b/e621/users.go similarity index 95% rename from e621/api/users.go rename to e621/users.go index 4416234..c35650a 100644 --- a/e621/api/users.go +++ b/e621/users.go @@ -1,7 +1,7 @@ -package api +package e621 import ( - "e621_to_neo4j/e621/api/models" + "e621_to_neo4j/e621/models" "fmt" "io" "net/http" diff --git a/main.go b/main.go index f2d1c63..ada29c9 100644 --- a/main.go +++ b/main.go @@ -2,21 +2,26 @@ package main import ( "context" - "e621_to_neo4j/e621/api" - "e621_to_neo4j/e621/api/models" + "e621_to_neo4j/api" + "e621_to_neo4j/e621" "e621_to_neo4j/neo4jAPI" "e621_to_neo4j/utils" "github.com/neo4j/neo4j-go-driver/v5/neo4j" "log" - "time" + "net/http" ) +const USER = "selloo" + func main() { + + // Loads Config config, err := utils.LoadConfig() if err != nil { log.Println(err) } + // Connect to Neo4j DB driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword) if err != nil { log.Fatal(err) @@ -30,215 +35,16 @@ func main() { } }(driver, ctx) - e621Client := api.NewClient(config.E621APIKey, config.E621Username) - user, err := e621Client.GetUserInfo("selloo") + // Initialize the e621API + e621Client := e621.NewClient(config.E621APIKey, config.E621Username) + + // Register the UserHandler with the "/user" route + http.HandleFunc("/user", api.UserHandler(ctx, driver, e621Client)) + + // Start the HTTP server + err = http.ListenAndServe(":8080", nil) if err != nil { - log.Fatal(err) + return } - err = neo4jAPI.CreateUserNode(ctx, driver, user) - if err != nil { - log.Fatal(err) - } - - favorites, err := e621Client.GetFavorites(user.Name) - if err != nil { - log.Fatal(err) - } - - start := time.Now() - //uploadTags(favorites, ctx, driver) - elapsed := time.Since(start) - log.Printf("upload of Tags took %d", elapsed) - - start = time.Now() - //uploadPosts(favorites, ctx, driver) - elapsed = time.Since(start) - log.Printf("upload of Posts took %d", elapsed) - - start = time.Now() - //uploadSources(favorites, ctx, driver) - elapsed = time.Since(start) - log.Printf("upload of Sources took %d", elapsed) - - start = time.Now() - // err = makePostToTagRelationship(favorites, ctx, driver) - elapsed = time.Since(start) - log.Printf("upload of Sources took %v", elapsed) - - start = time.Now() - // makeUserToPostRelationship(favorites, user, ctx, driver) - elapsed = time.Since(start) - log.Printf("upload of Sources took %v", elapsed) - - start = time.Now() - err = makePostToSourceRelationship(favorites, ctx, driver) - if err != nil { - log.Fatal(err) - } - elapsed = time.Since(start) - log.Printf("upload of Sources took %v", elapsed) - -} -func uploadPosts(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { - for _, fav := range posts { - log.Printf("e621 PostID: %d", fav.ID) - err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID) - if err != nil { - return err - } - - } - return nil - -} - -func uploadTags(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { - uniqueGeneralTags := make([]string, 0) - uniqueCharacterTags := make([]string, 0) - uniqueCopyrightTags := make([]string, 0) - uniqueArtistTags := make([]string, 0) - - allGeneralTags := make([]string, 0) - allCharacterTags := make([]string, 0) - allCopyrightTags := make([]string, 0) - allArtistTags := make([]string, 0) - - // add all tags together - for _, fav := range posts { - allGeneralTags = append(allGeneralTags, fav.Tags.General...) - allCharacterTags = append(allCharacterTags, fav.Tags.Character...) - allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...) - allArtistTags = append(allArtistTags, fav.Tags.Artist...) - } - - // Process General PostTags - uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) - - // Process Character PostTags - uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) - - // Process Copyright PostTags - uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) - - // Process Artist PostTags - uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) - - log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) - - log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) - - log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) - - for _, uniqueGeneralTag := range uniqueGeneralTags { - - log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") - if err != nil { - return err - } - } - for _, uniqueCharacterTag := range uniqueCharacterTags { - log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") - if err != nil { - return err - } - } - - for _, uniqueCopyrightTag := range uniqueCopyrightTags { - log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") - if err != nil { - return err - } - } - - for _, uniqueArtistTag := range uniqueArtistTags { - log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") - if err != nil { - return err - } - } - return nil -} - -func uploadSources(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { - for _, fav := range posts { - for _, source := range fav.Sources { - log.Printf("e621 Source: %s", source) - err := neo4jAPI.CreateSourceNode(ctx, driver, source) - if err != nil { - return err - } - } - } - return nil -} - -func makePostToTagRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { - - for _, fav := range posts { - id := fav.ID - for _, generalTags := range fav.Tags.General { - log.Printf("PostID: %d has General Tag: %s", id, generalTags) - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTags) - if err != nil { - return err - } - } - for _, characterTags := range fav.Tags.Character { - log.Printf("PostID: %d has Character Tag: %s", id, characterTags) - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTags) - if err != nil { - return err - } - } - for _, copyrightTags := range fav.Tags.Copyright { - log.Printf("PostID: %d has Copyright Tag: %s", id, copyrightTags) - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTags) - if err != nil { - return err - } - } - for _, artistTags := range fav.Tags.Artist { - log.Printf("PostID: %d has Artist Tag: %s", id, artistTags) - err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTags) - if err != nil { - return err - } - } - } - return nil -} - -func makeUserToPostRelationship(posts []models.Post, user models.E621User, ctx context.Context, driver neo4j.DriverWithContext) error { - - for _, post := range posts { - log.Printf("UserID: %d has PostID: %d in his favorites", user.ID, post.ID) - err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, user.ID) - if err != nil { - return err - } - } - - return nil -} - -func makePostToSourceRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { - - for _, post := range posts { - postID := post.ID - for _, source := range post.Sources { - log.Printf("PostID: %d has Source URL: %s", postID, source) - err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source) - if err != nil { - return err - } - } - - } - - return nil } diff --git a/neo4jAPI/tag.go b/neo4jAPI/tag.go index 8679835..8b5fa34 100644 --- a/neo4jAPI/tag.go +++ b/neo4jAPI/tag.go @@ -24,6 +24,31 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str return nil } +func CreateTagNode2(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error { + session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) + defer session.Close(ctx) + + query := ` + MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType}) + RETURN u +` + params := map[string]interface{}{ + "name": name, + "tagType": tagType, + } + + _, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (any, error) { + result, err := tx.Run(ctx, query, params) + if err != nil { + return nil, err + } + + return result.Consume(ctx) + }) + + return err +} + func GetAllTagNodes(ctx context.Context, driver neo4j.DriverWithContext, tagType string) ([]string, error) { query := ` MATCH (u:e621Tag {e621TagType: $tagType}) diff --git a/neo4jAPI/user.go b/neo4jAPI/user.go index 5a3073b..c5b223a 100644 --- a/neo4jAPI/user.go +++ b/neo4jAPI/user.go @@ -2,7 +2,7 @@ package neo4jAPI import ( "context" - "e621_to_neo4j/e621/api/models" + "e621_to_neo4j/e621/models" "github.com/neo4j/neo4j-go-driver/v5/neo4j" "log" ) diff --git a/services/user.go b/services/user.go new file mode 100644 index 0000000..4b893c4 --- /dev/null +++ b/services/user.go @@ -0,0 +1,153 @@ +package services + +import ( + "context" + "e621_to_neo4j/e621" + "e621_to_neo4j/neo4jAPI" + "e621_to_neo4j/utils" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + "log" + "time" +) + +func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error { + var err error + + e621User, err := e621Client.GetUserInfo(username) + if err != nil { + return err + } + + err = neo4jAPI.CreateUserNode(ctx, driver, e621User) + if err != nil { + log.Fatal(err) + } + + userFavorites, err := e621Client.GetFavorites(e621User.Name) + if err != nil { + log.Fatal(err) + } + + startUploadPosts := time.Now() + for i, post := range userFavorites { + start := time.Now() + + uniqueGeneralTags := make([]string, 0) + uniqueCharacterTags := make([]string, 0) + uniqueCopyrightTags := make([]string, 0) + uniqueArtistTags := make([]string, 0) + + allGeneralTags := make([]string, 0) + allCharacterTags := make([]string, 0) + allCopyrightTags := make([]string, 0) + allArtistTags := make([]string, 0) + + allGeneralTags = append(allGeneralTags, post.Tags.General...) + allCharacterTags = append(allCharacterTags, post.Tags.Character...) + allCopyrightTags = append(allCopyrightTags, post.Tags.Character...) + allArtistTags = append(allArtistTags, post.Tags.Artist...) + + uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) + uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) + uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) + uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) + + // Uploads post to database + err := neo4jAPI.CreatePostNode(ctx, driver, post.ID) + if err != nil { + return err + } + + // Uploads the source to the database + for _, source := range post.Sources { + err := neo4jAPI.CreateSourceNode(ctx, driver, source) + if err != nil { + return err + } + } + + for _, uniqueGeneralTag := range uniqueGeneralTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") + if err != nil { + return err + } + } + for _, uniqueCharacterTag := range uniqueCharacterTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") + if err != nil { + return err + } + } + + for _, uniqueCopyrightTag := range uniqueCopyrightTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") + if err != nil { + return err + } + } + + for _, uniqueArtistTag := range uniqueArtistTags { + err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") + if err != nil { + return err + } + } + log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start)) + } + + for _, post := range userFavorites { + id := post.ID + + err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID) + if err != nil { + return err + } + log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID) + + postID := post.ID + for _, source := range post.Sources { + err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source) + if err != nil { + return err + } + log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source) + + } + + for _, generalTag := range post.Tags.General { + err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag) + if err != nil { + log.Println(err) + } + log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag) + + } + for _, characterTag := range post.Tags.Character { + err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag) + if err != nil { + log.Println(err) + } + log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag) + + } + for _, copyrightTag := range post.Tags.Copyright { + err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag) + if err != nil { + log.Println(err) + } + log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag) + + } + for _, artistTag := range post.Tags.Artist { + err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag) + if err != nil { + log.Println(err) + } + log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag) + + } + } + + log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts)) + return nil +}