added DB Interface for later dgraph DB backend

api/user.go

@@ -2,15 +2,15 @@ package api
 
 import (
 	"context"
+	"e621_to_neo4j/database"
 	"e621_to_neo4j/e621"
 	"e621_to_neo4j/services"
 	"fmt"
-	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	"net/http"
 )
 
 // UserHandler is the handler for the user API
-func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
+func UserHandler(ctx context.Context, graphConnection database.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPost {
 			w.WriteHeader(http.StatusMethodNotAllowed)
@@ -26,7 +26,7 @@ func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client
 		}
 
 		// Perform further processing with the username
-		go services.ScrapeUser(ctx, driver, *e621Client, username)
+		go services.ScrapeUser(ctx, graphConnection, *e621Client, username)
 
 		// Send a response
 		w.WriteHeader(http.StatusOK)

database/dgraph/impl.go

@@ -0,0 +1,79 @@
+package dgraph
+
+import (
+	"context"
+	"e621_to_neo4j/database"
+	"e621_to_neo4j/e621/models"
+	"github.com/dgraph-io/dgo/v210"
+	"github.com/dgraph-io/dgo/v210/protos/api"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/credentials/insecure"
+	"google.golang.org/grpc/encoding/gzip"
+	"log"
+)
+
+type dgraphConnection struct {
+	driver dgo.Dgraph
+}
+
+func (d *dgraphConnection) UploadUser(ctx context.Context, user models.E621User) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (d *dgraphConnection) UploadSource(ctx context.Context, SourceURL string) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (d *dgraphConnection) UploadPost(ctx context.Context, e621ID int64) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (d *dgraphConnection) UploadTag(ctx context.Context, name string, tagType string) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (d *dgraphConnection) EstablishPostTagLink(ctx context.Context, e621PostID int64, e621Tag string) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (d *dgraphConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (d *dgraphConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error {
+	//TODO implement me
+	panic("implement me")
+}
+
+func (d *dgraphConnection) Connect(ctx context.Context, endpoint string, username string, password string) error {
+	// Dial a gRPC connection. The address to dial to can be configured when
+	// setting up the dgraph cluster.
+	dialOpts := append([]grpc.DialOption{},
+		grpc.WithTransportCredentials(insecure.NewCredentials()),
+		grpc.WithDefaultCallOptions(grpc.UseCompressor(gzip.Name)))
+	conn, err := grpc.Dial(endpoint, dialOpts...)
+
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	defer func(conn *grpc.ClientConn) {
+		err := conn.Close()
+		if err != nil {
+			return
+		}
+	}(conn)
+
+	d.driver = *dgo.NewDgraphClient(api.NewDgraphClient(conn))
+	return nil
+}
+
+func NewDGraphConnection() database.GraphConnection {
+	return &dgraphConnection{}
+}

database/logic.go

@@ -0,0 +1,27 @@
+package database
+
+import (
+	"context"
+	"e621_to_neo4j/e621/models"
+)
+
+// [X:Neo4J] Connection
+// [X:Neo4J] Upload User
+// [X:Neo4J] Upload Source
+// [X:Neo4J] Upload Post
+// [X:Neo4J] Upload Tag
+// [X:Neo4J] Relationship Post->Tag
+// [X:Neo4J] Relationship Post->Source
+// [X:Neo4J] Relationship User->Post
+
+type GraphConnection interface {
+	Connect(ctx context.Context, endpoint string, username string, password string) error
+	UploadUser(ctx context.Context, user models.E621User) error
+	UploadSource(ctx context.Context, SourceURL string) error
+	UploadPost(ctx context.Context, e621ID int64) error
+	UploadTag(ctx context.Context, name string, tagType string) error
+	EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error
+	EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error
+	EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error
+	CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error)
+}

database/neo4j/impl.go

@@ -0,0 +1,68 @@
+package neo4j
+
+import (
+	"context"
+	"e621_to_neo4j/database"
+	"e621_to_neo4j/e621/models"
+	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
+	"github.com/neo4j/neo4j-go-driver/v5/neo4j/config"
+)
+
+type neo4jConnection struct {
+	driver neo4j.DriverWithContext
+}
+
+func (c *neo4jConnection) CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error) {
+	return RelationshipCheckUserToPost(ctx, c.driver, e621PostID, e621UserID)
+}
+
+func (c *neo4jConnection) EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error {
+	return EstablishPostTagLink(ctx, c.driver, e621PostID, e621Tag)
+}
+
+func (c *neo4jConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error {
+	return EstablishPostToSourceLink(ctx, c.driver, e621PostID, sourceURL)
+}
+
+func (c *neo4jConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error {
+	return EstablishUserToPostLink(ctx, c.driver, e621PostID, e621UserID)
+}
+
+func (c *neo4jConnection) UploadTag(ctx context.Context, name string, tagType string) error {
+	return CreateTagNode(ctx, c.driver, name, tagType)
+}
+
+func (c *neo4jConnection) UploadPost(ctx context.Context, e621ID int64) error {
+	return CreatePostNode(ctx, c.driver, e621ID)
+}
+
+func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) error {
+	return CreateSourceNode(ctx, c.driver, SourceURL)
+}
+
+func (c *neo4jConnection) UploadUser(ctx context.Context, user models.E621User) error {
+	return CreateUserNode(ctx, c.driver, user)
+}
+func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error {
+	driver, err := neo4j.NewDriverWithContext(endpoint, neo4j.BasicAuth(username, password, ""),
+		useConsoleLogger(neo4j.INFO))
+	if err != nil {
+		return err
+	}
+	err = driver.VerifyAuthentication(context.Background(), nil)
+	if err != nil {
+		return err
+	}
+	c.driver = driver
+	return nil
+}
+
+func NewNeo4JConnection() database.GraphConnection {
+	return &neo4jConnection{}
+}
+
+func useConsoleLogger(level neo4j.LogLevel) func(config *config.Config) {
+	return func(config *config.Config) {
+		config.Log = neo4j.ConsoleLogger(level)
+	}
+}

database/neo4j/post.go

@@ -1,4 +1,4 @@
-package neo4jAPI
+package neo4j
 
 import (
 	"context"

database/neo4j/relationship.go

@@ -1,11 +1,11 @@
-package neo4jAPI
+package neo4j
 
 import (
 	"context"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 )
 
-func PostToTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error {
+func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error {
 	query := `
 		MATCH (p:e621Post {e621PostID: $e621PostID})
 		MATCH (t:e621Tag {e621Tag: $e621Tag})
@@ -24,7 +24,7 @@ func PostToTagRelationship(ctx context.Context, driver neo4j.DriverWithContext,
 	return nil
 }
 
-func PostToSourceRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error {
+func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error {
 	query := `
 		MATCH (p:e621Post {e621PostID: $e621PostID})
 		MATCH (s:Source {URL: $sourceURL})
@@ -43,7 +43,7 @@ func PostToSourceRelationship(ctx context.Context, driver neo4j.DriverWithContex
 	return nil
 }
 
-func UserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error {
+func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error {
 	query := `
 		MATCH (p:e621Post {e621PostID: $e621PostID})
 		MATCH (u:e621User {e621ID: $e621ID})
@@ -63,7 +63,7 @@ func UserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext,
 }
 
 // CheckUserToPostRelationship gives back a bool if the connection between the post and the user exists
-func CheckUserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) {
+func RelationshipCheckUserToPost(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) {
 	query := `
 		MATCH (user:e621User {e621ID: $e621ID})-[favorite:IS_FAVORITE]->(post:e621Post {e621PostID: $e621PostID})
 		RETURN COUNT(favorite) > 0 AS isFavorite

database/neo4j/source.go

@@ -1,4 +1,4 @@
-package neo4jAPI
+package neo4j
 
 import (
 	"context"

database/neo4j/tag.go

@@ -1,8 +1,8 @@
-package neo4jAPI
+package neo4j
 
 import (
 	"context"
-	"e621_to_neo4j/neo4jAPI/models"
+	"e621_to_neo4j/database/neo4j/models"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 )
 
@@ -24,55 +24,6 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str
 	return nil
 }
 
-func CreateTagNode2(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error {
-	session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
-	defer session.Close(ctx)
-
-	query := `
-		MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType})
-		RETURN u
-`
-	params := map[string]interface{}{
-		"name":    name,
-		"tagType": tagType,
-	}
-
-	_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (any, error) {
-		result, err := tx.Run(ctx, query, params)
-		if err != nil {
-			return nil, err
-		}
-
-		return result.Consume(ctx)
-	})
-
-	return err
-}
-
-func GetAllTagNodes(ctx context.Context, driver neo4j.DriverWithContext, tagType string) ([]string, error) {
-	query := `
-		MATCH (u:e621Tag {e621TagType: $tagType})
-		RETURN u.e621Tag as eTag
-	`
-
-	params := map[string]interface{}{
-		"tagType": tagType,
-	}
-
-	result, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)
-	if err != nil {
-		return nil, err
-	}
-
-	var tags []string
-	for _, record := range result.Records {
-		tag, _, _ := neo4j.GetRecordValue[string](record, "eTag")
-		tags = append(tags, tag)
-	}
-
-	return tags, nil
-}
-
 func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string) (models.DBTag, bool, error) {
 
 	var tag models.DBTag

database/neo4j/user.go

@@ -1,4 +1,4 @@
-package neo4jAPI
+package neo4j
 
 import (
 	"context"

docker-compose.dgraph.yml

@@ -2,8 +2,6 @@ version: "3.2"
 services:
   zero:
     image: dgraph/dgraph:latest
-    volumes:
-      - /tmp/data:/dgraph
     ports:
       - 5080:5080
       - 6080:6080
@@ -11,8 +9,6 @@ services:
     command: dgraph zero --my=zero:5080
   alpha:
     image: dgraph/dgraph:latest
-    volumes:
-      - /tmp/data:/dgraph
     ports:
       - 8080:8080
       - 9080:9080

e621/client.go

@@ -1,6 +1,7 @@
 package e621
 
 import (
+	"e621_to_neo4j/utils"
 	"golang.org/x/time/rate"
 	"net/http"
 )
@@ -15,6 +16,7 @@ type Client struct {
 	username string
 	client   *http.Client
 	limiter  *rate.Limiter
+	queue    *utils.Queue
 }
 
 // NewClient creates a new e621 API client.
@@ -24,5 +26,6 @@ func NewClient(apiKey string, username string) *Client {
 		username: username,
 		client:   &http.Client{},
 		limiter:  rate.NewLimiter(1, 2),
+		queue:    &utils.Queue{},
 	}
 }

e621/favorite.go

@@ -4,19 +4,17 @@ import (
 	"e621_to_neo4j/e621/models"
 	"fmt"
 	"log"
-	"time"
 )
 
 // GetFavorites retrieves all favorites from the e621 API.
 func (c *Client) GetFavorites(user models.E621User) ([]models.Post, error) {
-	time.Sleep(1 * time.Second)
+	page := 1
-	var page int64
 	var allFavorites []models.Post
 	var URIPath string
 
 	for {
 		URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page)
-		favorite, err := ExecuteGetAPIRequest[models.APIe621Post](c, URIPath)
+		favorite, err := ExecuteGetAPIRequest[models.PostResponseWrapper](c, URIPath)
 		if err != nil {
 			log.Printf(err.Error())
 		}

e621/models/post.go

@@ -2,17 +2,17 @@ package models
 
 import "encoding/json"
 
-func UnmarshalE621Post(data []byte) (APIe621Post, error) {
+func UnmarshalE621Post(data []byte) (PostResponseWrapper, error) {
-	var r APIe621Post
+	var r PostResponseWrapper
 	err := json.Unmarshal(data, &r)
 	return r, err
 }
 
-func (r *APIe621Post) Marshal() ([]byte, error) {
+func (r *PostResponseWrapper) Marshal() ([]byte, error) {
 	return json.Marshal(r)
 }
 
-type APIe621Post struct {
+type PostResponseWrapper struct {
 	Posts []Post `json:"posts"`
 }
 

e621/request.go

@@ -5,12 +5,14 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"log"
 	"net/http"
 )
 
 func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) {
 	var err error
 	c.limiter.Wait(context.Background())
+	log.Println(URIPath)
 	url := fmt.Sprintf("%s/%s", baseURL, URIPath)
 	req, err := http.NewRequest("GET", url, nil)
 	if err != nil {

go.mod

@@ -9,7 +9,18 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/stretchr/testify v1.5.1 // indirect
+	golang.org/x/net v0.11.0 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/text v0.10.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
+	google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/grpc v1.56.0 // indirect
+	google.golang.org/protobuf v1.30.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -3,16 +3,71 @@ github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2Uo
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d h1:abDbP7XBVgwda+h0J5Qra5p2OQpidU2FdkXvzCKL+H8=
+github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d/go.mod h1:wKFzULXAPj3U2BDAPWXhSbQQNC6FU1+1/5iika6IY7g=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/neo4j/neo4j-go-driver/v5 v5.8.1 h1:IysKg6KJIUgyItmnHRRrt2N8srbd6znMslRW3qQErTQ=
 github.com/neo4j/neo4j-go-driver/v5 v5.8.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
+golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
+golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
+golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
 golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
 golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 h1:x1vNwUhVOcsYoKyEGCZBH694SBmmBjA2EfauFVEI2+M=
+google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
+google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE=
+google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
+google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -3,15 +3,18 @@ package main
 import (
 	"context"
 	"e621_to_neo4j/api"
+	"e621_to_neo4j/database"
+	neo4j "e621_to_neo4j/database/neo4j"
 	"e621_to_neo4j/e621"
-	"e621_to_neo4j/neo4jAPI"
 	"e621_to_neo4j/utils"
-	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	"log"
 	"net/http"
+	"strings"
 )
 
 func main() {
+	var graphConnection database.GraphConnection
+	ctx := context.Background()
 
 	// Loads Config
 	config, err := utils.LoadConfig()
@@ -19,29 +22,18 @@ func main() {
 		log.Println(err)
 	}
 
+	switch strings.ToLower(config.DBType) {
+	case "neo4j":
 		log.Println("Setup Neo4J Connection")
-	// Connect to Neo4j DB
+		graphConnection = neo4j.NewNeo4JConnection()
-	driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
+		err = graphConnection.Connect(ctx, config.DBEndpoint, config.Neo4jUsername, config.Neo4jPassword)
 		if err != nil {
-		log.Fatal(err)
+			panic(err)
 		}
-
-	err = driver.VerifyAuthentication(context.Background(), nil)
-
-	if err != nil {
-		log.Println("Cannot connect and authenticate to NEO4J DB")
-		log.Fatal(err)
-	}
-
 		log.Println("Connection successful")
-
+	default:
-	ctx := context.Background()
+		panic("No Database was selected!")
-	defer func(driver neo4j.DriverWithContext, ctx context.Context) {
-		err := driver.Close(ctx)
-		if err != nil {
-			log.Fatal(err)
 	}
-	}(driver, ctx)
 
 	// Initialize the e621API
 	e621Client := e621.NewClient(config.E621APIKey, config.E621Username)
@@ -49,7 +41,7 @@ func main() {
 	log.Printf("Im ready!")
 
 	// Register the UserHandler with the "/user" route
-	http.HandleFunc("/user", api.UserHandler(ctx, driver, e621Client))
+	http.HandleFunc("/user", api.UserHandler(ctx, graphConnection, e621Client))
 
 	// Start the HTTP server
 	err = http.ListenAndServe(":8080", nil)

neo4jAPI/connection.go

@@ -1,23 +0,0 @@
-package neo4jAPI
-
-import (
-	"fmt"
-	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
-	"github.com/neo4j/neo4j-go-driver/v5/neo4j/config"
-)
-
-func NewConnection(uri string, username string, password string) (neo4j.DriverWithContext, error) {
-	driver, err := neo4j.NewDriverWithContext(uri, neo4j.BasicAuth(username, password, ""),
-		useConsoleLogger(neo4j.INFO))
-	if err != nil {
-		return nil, fmt.Errorf("failed to create Neo4j driver: %v", err)
-	}
-	return driver, nil
-
-}
-
-func useConsoleLogger(level neo4j.LogLevel) func(config *config.Config) {
-	return func(config *config.Config) {
-		config.Log = neo4j.ConsoleLogger(level)
-	}
-}

services/user.go

@@ -2,16 +2,15 @@ package services
 
 import (
 	"context"
+	"e621_to_neo4j/database"
 	"e621_to_neo4j/e621"
 	"e621_to_neo4j/e621/models"
-	"e621_to_neo4j/neo4jAPI"
 	"e621_to_neo4j/utils"
-	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	"log"
 	"time"
 )
 
-func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error {
+func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e621Client e621.Client, username string) error {
 	var err error
 
 	e621User, err := e621Client.GetUserInfo(username)
@@ -26,7 +25,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
 
 	log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID)
 
-	err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
+	err = graphConnection.UploadUser(ctx, e621User)
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -43,8 +42,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
 
 	// Uploads all Tags, Posts as Nodes to Neo4j
 	for i, post := range userFavorites {
-
+		if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists {
-		if exists, err := neo4jAPI.CheckUserToPostRelationship(ctx, driver, post.ID, e621User.ID); err == nil && exists {
 			log.Printf("No new posts found for user %s with id %d", e621User.Name, e621User.ID)
 			log.Printf("Last Post ID Found: %d", post.ID)
 			break
@@ -53,44 +51,44 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
 		}
 
 		start = time.Now()
-		err = uploadNodes(ctx, driver, post)
+		err = uploadNodes(ctx, graphConnection, post)
 		if err != nil {
 			return err
 		}
 		log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))
 
 		start := time.Now()
-		err = uploadPostToUserRelationship(ctx, driver, post, e621User)
+		err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User)
 		if err != nil {
 			log.Fatal(err)
 			return err
 		}
 
-		err = uploadSourceTagRelationship(ctx, driver, post)
+		err = uploadSourceTagRelationship(ctx, graphConnection, post)
 		if err != nil {
 			log.Fatal(err)
 			return err
 		}
 
-		err = uploadGeneralTagRelationship(ctx, driver, post)
+		err = uploadGeneralTagRelationship(ctx, graphConnection, post)
 		if err != nil {
 			log.Fatal(err)
 			return err
 		}
 
-		err = uploadCharacterTagtRelationship(ctx, driver, post)
+		err = uploadCharacterTagtRelationship(ctx, graphConnection, post)
 		if err != nil {
 			log.Fatal(err)
 			return err
 		}
 
-		err = uploadCopyrightTagRelationship(ctx, driver, post)
+		err = uploadCopyrightTagRelationship(ctx, graphConnection, post)
 		if err != nil {
 			log.Fatal(err)
 			return err
 		}
 
-		err = uploadArtistTagRelationship(ctx, driver, post)
+		err = uploadArtistTagRelationship(ctx, graphConnection, post)
 		if err != nil {
 			log.Fatal(err)
 			return err
@@ -103,7 +101,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
 }
 
 // uploadNodes uploads the post to the database and creates the nodes
-func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
+func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
 
 	uniqueGeneralTags := make([]string, 0)
 	uniqueCharacterTags := make([]string, 0)
@@ -125,41 +123,41 @@ func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post model
 	uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
 	uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
 
-	err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
+	err := graphConnection.UploadPost(ctx, post.ID)
 	if err != nil {
 		return err
 	}
 
 	// Uploads the source to the database
 	for _, source := range post.Sources {
-		err := neo4jAPI.CreateSourceNode(ctx, driver, source)
+		err := graphConnection.UploadSource(ctx, source)
 		if err != nil {
 			return err
 		}
 	}
 
 	for _, uniqueGeneralTag := range uniqueGeneralTags {
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
+		err := graphConnection.UploadTag(ctx, uniqueGeneralTag, "general")
 		if err != nil {
 			return err
 		}
 	}
 	for _, uniqueCharacterTag := range uniqueCharacterTags {
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
+		err := graphConnection.UploadTag(ctx, uniqueCharacterTag, "character")
 		if err != nil {
 			return err
 		}
 	}
 
 	for _, uniqueCopyrightTag := range uniqueCopyrightTags {
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
+		err := graphConnection.UploadTag(ctx, uniqueCopyrightTag, "copyright")
 		if err != nil {
 			return err
 		}
 	}
 
 	for _, uniqueArtistTag := range uniqueArtistTags {
-		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
+		err := graphConnection.UploadTag(ctx, uniqueArtistTag, "artist")
 		if err != nil {
 			return err
 		}
@@ -169,8 +167,8 @@ func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post model
 }
 
 // uploadPostToUserRelationship creates a relationship between the user and the post
-func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post, e621User models.E621User) error {
+func uploadPostToUserRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post, e621User models.E621User) error {
-	err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
+	err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID)
 	if err != nil {
 		return err
 	}
@@ -179,9 +177,9 @@ func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithCo
 }
 
 // uploadSourceTagRelationship creates a relationship between the post and the source
-func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
+func uploadSourceTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
 	for _, source := range post.Sources {
-		err := neo4jAPI.PostToSourceRelationship(ctx, driver, post.ID, source)
+		err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source)
 		if err != nil {
 			return err
 		}
@@ -192,9 +190,9 @@ func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithCon
 }
 
 // uploadGeneralTagRelationship creates a relationship between the post and the general tag
-func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
+func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
 	for _, generalTag := range post.Tags.General {
-		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, generalTag)
+		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag)
 		if err != nil {
 			return err
 		}
@@ -205,9 +203,9 @@ func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithCo
 }
 
 // uploadCharacterTagtRelationship creates a relationship between the post and the character tag
-func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
+func uploadCharacterTagtRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
 	for _, characterTag := range post.Tags.Character {
-		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, characterTag)
+		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag)
 		if err != nil {
 			return err
 		}
@@ -218,9 +216,9 @@ func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWit
 }
 
 // uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag
-func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
+func uploadCopyrightTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
 	for _, copyrightTag := range post.Tags.Copyright {
-		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, copyrightTag)
+		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag)
 		if err != nil {
 			return err
 		}
@@ -231,9 +229,9 @@ func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWith
 }
 
 // uploadArtistTagRelationship creates a relationship between the post and the artist tag
-func uploadArtistTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
+func uploadArtistTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
 	for _, artistTag := range post.Tags.Artist {
-		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, artistTag)
+		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag)
 		if err != nil {
 			return err
 		}

utils/config.go

@@ -8,7 +8,8 @@ import (
 type Config struct {
 	E621APIKey    string `env:"E621_API_KEY,required"`
 	E621Username  string `env:"E621_USERNAME,required"`
-	Neo4jURL      string `env:"NEO4J_URL,required"`
+	DBType        string `env:"DB_TYPE,required"`
+	DBEndpoint    string `env:"DB_URL,required"`
 	Neo4jUsername string `env:"NEO4J_USERNAME,required"`
 	Neo4jPassword string `env:"NEO4J_PASSWORD,required"`
 }