added user api implementation

api/user.go

@@ -0,0 +1,36 @@
+package api
+
+import (
+	"context"
+	"e621_to_neo4j/e621"
+	"e621_to_neo4j/services"
+	"fmt"
+	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
+	"net/http"
+)
+
+// UserHandler is the handler for the user API
+func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodPost {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			fmt.Fprintf(w, "Only POST requests are allowed")
+			return
+		}
+
+		username := r.FormValue("username")
+		if username == "" {
+			w.WriteHeader(http.StatusBadRequest)
+			fmt.Fprintf(w, "Username is required")
+			return
+		}
+
+		// Perform further processing with the username
+		services.ScrapeUser(ctx, driver, *e621Client, username)
+
+		// Send a response
+		w.WriteHeader(http.StatusOK)
+		fmt.Fprintf(w, "Username %s processed successfully", username)
+
+	}
+}

docker-compose.yml

@@ -6,3 +6,8 @@ services:
       - '7474:7474'
       - '7473:7473'
       - '7687:7687'
+    volumes:
+      - 'neo4j_data:/bitnami'
+volumes:
+  neo4j_data:
+    driver: local

e621/client.go

@@ -1,4 +1,4 @@
-package api
+package e621
 
 import (
 	"net/http"

e621/favorite.go

@@ -1,7 +1,7 @@
-package api
+package e621
 
 import (
-	"e621_to_neo4j/e621/api/models"
+	"e621_to_neo4j/e621/models"
 	"fmt"
 	"io"
 	"log"

e621/models/post.go

@@ -2,17 +2,17 @@ package models
 
 import "encoding/json"
 
-func UnmarshalE621Post(data []byte) (E621Post, error) {
+func UnmarshalE621Post(data []byte) (APIe621Post, error) {
-	var r E621Post
+	var r APIe621Post
 	err := json.Unmarshal(data, &r)
 	return r, err
 }
 
-func (r *E621Post) Marshal() ([]byte, error) {
+func (r *APIe621Post) Marshal() ([]byte, error) {
 	return json.Marshal(r)
 }
 
-type E621Post struct {
+type APIe621Post struct {
 	Posts []Post `json:"posts"`
 }
 

e621/users.go

@@ -1,7 +1,7 @@
-package api
+package e621
 
 import (
-	"e621_to_neo4j/e621/api/models"
+	"e621_to_neo4j/e621/models"
 	"fmt"
 	"io"
 	"net/http"

main.go

@@ -2,21 +2,26 @@ package main
 
 import (
 	"context"
-	"e621_to_neo4j/e621/api"
+	"e621_to_neo4j/api"
-	"e621_to_neo4j/e621/api/models"
+	"e621_to_neo4j/e621"
 	"e621_to_neo4j/neo4jAPI"
 	"e621_to_neo4j/utils"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	"log"
-	"time"
+	"net/http"
 )
 
+const USER = "selloo"
+
 func main() {
+
+	// Loads Config
 	config, err := utils.LoadConfig()
 	if err != nil {
 		log.Println(err)
 	}
 
+	// Connect to Neo4j DB
 	driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
 	if err != nil {
 		log.Fatal(err)
@@ -30,215 +35,16 @@ func main() {
 		}
 	}(driver, ctx)
 
-	e621Client := api.NewClient(config.E621APIKey, config.E621Username)
+	// Initialize the e621API
-	user, err := e621Client.GetUserInfo("selloo")
+	e621Client := e621.NewClient(config.E621APIKey, config.E621Username)
+
+	// Register the UserHandler with the "/user" route
+	http.HandleFunc("/user", api.UserHandler(ctx, driver, e621Client))
+
+	// Start the HTTP server
+	err = http.ListenAndServe(":8080", nil)
 	if err != nil {
-		log.Fatal(err)
+		return
-	}
-
-	err = neo4jAPI.CreateUserNode(ctx, driver, user)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	favorites, err := e621Client.GetFavorites(user.Name)
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	start := time.Now()
-	//uploadTags(favorites, ctx, driver)
-	elapsed := time.Since(start)
-	log.Printf("upload of Tags took %d", elapsed)
-
-	start = time.Now()
-	//uploadPosts(favorites, ctx, driver)
-	elapsed = time.Since(start)
-	log.Printf("upload of Posts took %d", elapsed)
-
-	start = time.Now()
-	//uploadSources(favorites, ctx, driver)
-	elapsed = time.Since(start)
-	log.Printf("upload of Sources took %d", elapsed)
-
-	start = time.Now()
-	// err = makePostToTagRelationship(favorites, ctx, driver)
-	elapsed = time.Since(start)
-	log.Printf("upload of Sources took %v", elapsed)
-
-	start = time.Now()
-	// makeUserToPostRelationship(favorites, user, ctx, driver)
-	elapsed = time.Since(start)
-	log.Printf("upload of Sources took %v", elapsed)
-
-	start = time.Now()
-	err = makePostToSourceRelationship(favorites, ctx, driver)
-	if err != nil {
-		log.Fatal(err)
-	}
-	elapsed = time.Since(start)
-	log.Printf("upload of Sources took %v", elapsed)
-
-}
-func uploadPosts(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
-	for _, fav := range posts {
-		log.Printf("e621 PostID: %d", fav.ID)
-		err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID)
-		if err != nil {
-			return err
 	}
 
 }
-	return nil
-
-}
-
-func uploadTags(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
-	uniqueGeneralTags := make([]string, 0)
-	uniqueCharacterTags := make([]string, 0)
-	uniqueCopyrightTags := make([]string, 0)
-	uniqueArtistTags := make([]string, 0)
-
-	allGeneralTags := make([]string, 0)
-	allCharacterTags := make([]string, 0)
-	allCopyrightTags := make([]string, 0)
-	allArtistTags := make([]string, 0)
-
-	// add all tags together
-	for _, fav := range posts {
-		allGeneralTags = append(allGeneralTags, fav.Tags.General...)
-		allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
-		allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
-		allArtistTags = append(allArtistTags, fav.Tags.Artist...)
-	}
-
-	// Process General PostTags
-	uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
-
-	// Process Character PostTags
-	uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
-
-	// Process Copyright PostTags
-	uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
-
-	// Process Artist PostTags
-	uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
-
-	log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
-
-	log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
-
-	log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
-
-	for _, uniqueGeneralTag := range uniqueGeneralTags {
-
-		log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
-		if err != nil {
-			return err
-		}
-	}
-	for _, uniqueCharacterTag := range uniqueCharacterTags {
-		log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
-		if err != nil {
-			return err
-		}
-	}
-
-	for _, uniqueCopyrightTag := range uniqueCopyrightTags {
-		log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
-		if err != nil {
-			return err
-		}
-	}
-
-	for _, uniqueArtistTag := range uniqueArtistTags {
-		log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
-		if err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-func uploadSources(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
-	for _, fav := range posts {
-		for _, source := range fav.Sources {
-			log.Printf("e621 Source: %s", source)
-			err := neo4jAPI.CreateSourceNode(ctx, driver, source)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func makePostToTagRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
-
-	for _, fav := range posts {
-		id := fav.ID
-		for _, generalTags := range fav.Tags.General {
-			log.Printf("PostID: %d has General Tag: %s", id, generalTags)
-			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTags)
-			if err != nil {
-				return err
-			}
-		}
-		for _, characterTags := range fav.Tags.Character {
-			log.Printf("PostID: %d has Character Tag: %s", id, characterTags)
-			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTags)
-			if err != nil {
-				return err
-			}
-		}
-		for _, copyrightTags := range fav.Tags.Copyright {
-			log.Printf("PostID: %d has Copyright Tag: %s", id, copyrightTags)
-			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTags)
-			if err != nil {
-				return err
-			}
-		}
-		for _, artistTags := range fav.Tags.Artist {
-			log.Printf("PostID: %d has Artist Tag: %s", id, artistTags)
-			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTags)
-			if err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func makeUserToPostRelationship(posts []models.Post, user models.E621User, ctx context.Context, driver neo4j.DriverWithContext) error {
-
-	for _, post := range posts {
-		log.Printf("UserID: %d has PostID: %d in his favorites", user.ID, post.ID)
-		err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, user.ID)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func makePostToSourceRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
-
-	for _, post := range posts {
-		postID := post.ID
-		for _, source := range post.Sources {
-			log.Printf("PostID: %d has Source URL: %s", postID, source)
-			err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
-			if err != nil {
-				return err
-			}
-		}
-
-	}
-
-	return nil
-}

neo4jAPI/tag.go

@@ -24,6 +24,31 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str
 	return nil
 }
 
+func CreateTagNode2(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error {
+	session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
+	defer session.Close(ctx)
+
+	query := `
+		MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType})
+		RETURN u
+`
+	params := map[string]interface{}{
+		"name":    name,
+		"tagType": tagType,
+	}
+
+	_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (any, error) {
+		result, err := tx.Run(ctx, query, params)
+		if err != nil {
+			return nil, err
+		}
+
+		return result.Consume(ctx)
+	})
+
+	return err
+}
+
 func GetAllTagNodes(ctx context.Context, driver neo4j.DriverWithContext, tagType string) ([]string, error) {
 	query := `
 		MATCH (u:e621Tag {e621TagType: $tagType})

neo4jAPI/user.go

@@ -2,7 +2,7 @@ package neo4jAPI
 
 import (
 	"context"
-	"e621_to_neo4j/e621/api/models"
+	"e621_to_neo4j/e621/models"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	"log"
 )

services/user.go

@@ -0,0 +1,153 @@
+package services
+
+import (
+	"context"
+	"e621_to_neo4j/e621"
+	"e621_to_neo4j/neo4jAPI"
+	"e621_to_neo4j/utils"
+	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
+	"log"
+	"time"
+)
+
+func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error {
+	var err error
+
+	e621User, err := e621Client.GetUserInfo(username)
+	if err != nil {
+		return err
+	}
+
+	err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	userFavorites, err := e621Client.GetFavorites(e621User.Name)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	startUploadPosts := time.Now()
+	for i, post := range userFavorites {
+		start := time.Now()
+
+		uniqueGeneralTags := make([]string, 0)
+		uniqueCharacterTags := make([]string, 0)
+		uniqueCopyrightTags := make([]string, 0)
+		uniqueArtistTags := make([]string, 0)
+
+		allGeneralTags := make([]string, 0)
+		allCharacterTags := make([]string, 0)
+		allCopyrightTags := make([]string, 0)
+		allArtistTags := make([]string, 0)
+
+		allGeneralTags = append(allGeneralTags, post.Tags.General...)
+		allCharacterTags = append(allCharacterTags, post.Tags.Character...)
+		allCopyrightTags = append(allCopyrightTags, post.Tags.Character...)
+		allArtistTags = append(allArtistTags, post.Tags.Artist...)
+
+		uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
+		uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
+		uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
+		uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
+
+		// Uploads post to database
+		err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
+		if err != nil {
+			return err
+		}
+
+		// Uploads the source to the database
+		for _, source := range post.Sources {
+			err := neo4jAPI.CreateSourceNode(ctx, driver, source)
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, uniqueGeneralTag := range uniqueGeneralTags {
+			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
+			if err != nil {
+				return err
+			}
+		}
+		for _, uniqueCharacterTag := range uniqueCharacterTags {
+			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, uniqueCopyrightTag := range uniqueCopyrightTags {
+			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
+			if err != nil {
+				return err
+			}
+		}
+
+		for _, uniqueArtistTag := range uniqueArtistTags {
+			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
+			if err != nil {
+				return err
+			}
+		}
+		log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start))
+	}
+
+	for _, post := range userFavorites {
+		id := post.ID
+
+		err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
+		if err != nil {
+			return err
+		}
+		log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID)
+
+		postID := post.ID
+		for _, source := range post.Sources {
+			err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
+			if err != nil {
+				return err
+			}
+			log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source)
+
+		}
+
+		for _, generalTag := range post.Tags.General {
+			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag)
+			if err != nil {
+				log.Println(err)
+			}
+			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag)
+
+		}
+		for _, characterTag := range post.Tags.Character {
+			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag)
+			if err != nil {
+				log.Println(err)
+			}
+			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag)
+
+		}
+		for _, copyrightTag := range post.Tags.Copyright {
+			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag)
+			if err != nil {
+				log.Println(err)
+			}
+			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag)
+
+		}
+		for _, artistTag := range post.Tags.Artist {
+			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag)
+			if err != nil {
+				log.Println(err)
+			}
+			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag)
+
+		}
+	}
+
+	log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts))
+	return nil
+}