From f95e9b0853f8d306633cd87a7256e0453e823913 Mon Sep 17 00:00:00 2001 From: David Janowski Date: Tue, 20 Jun 2023 10:38:36 +0200 Subject: [PATCH] added DB Interface for later dgraph DB backend --- api/user.go | 6 +- database/dgraph/impl.go | 79 ++++++++++++++++++++ database/logic.go | 27 +++++++ database/neo4j/impl.go | 68 +++++++++++++++++ {neo4jAPI => database/neo4j}/models/tag.go | 0 {neo4jAPI => database/neo4j}/post.go | 2 +- {neo4jAPI => database/neo4j}/relationship.go | 10 +-- {neo4jAPI => database/neo4j}/source.go | 2 +- {neo4jAPI => database/neo4j}/tag.go | 53 +------------ {neo4jAPI => database/neo4j}/user.go | 2 +- docker-compose.dgraph.yml | 4 - e621/client.go | 3 + e621/favorite.go | 6 +- e621/models/post.go | 8 +- e621/request.go | 2 + go.mod | 11 +++ go.sum | 55 ++++++++++++++ main.go | 40 ++++------ neo4jAPI/connection.go | 23 ------ services/user.go | 62 ++++++++------- utils/config.go | 3 +- 21 files changed, 312 insertions(+), 154 deletions(-) create mode 100644 database/dgraph/impl.go create mode 100644 database/logic.go create mode 100644 database/neo4j/impl.go rename {neo4jAPI => database/neo4j}/models/tag.go (100%) rename {neo4jAPI => database/neo4j}/post.go (96%) rename {neo4jAPI => database/neo4j}/relationship.go (80%) rename {neo4jAPI => database/neo4j}/source.go (95%) rename {neo4jAPI => database/neo4j}/tag.go (51%) rename {neo4jAPI => database/neo4j}/user.go (97%) delete mode 100644 neo4jAPI/connection.go diff --git a/api/user.go b/api/user.go index fa6be9f..2c8d261 100644 --- a/api/user.go +++ b/api/user.go @@ -2,15 +2,15 @@ package api import ( "context" + "e621_to_neo4j/database" "e621_to_neo4j/e621" "e621_to_neo4j/services" "fmt" - "github.com/neo4j/neo4j-go-driver/v5/neo4j" "net/http" ) // UserHandler is the handler for the user API -func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) { +func UserHandler(ctx context.Context, graphConnection database.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) @@ -26,7 +26,7 @@ func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client } // Perform further processing with the username - go services.ScrapeUser(ctx, driver, *e621Client, username) + go services.ScrapeUser(ctx, graphConnection, *e621Client, username) // Send a response w.WriteHeader(http.StatusOK) diff --git a/database/dgraph/impl.go b/database/dgraph/impl.go new file mode 100644 index 0000000..77cfe63 --- /dev/null +++ b/database/dgraph/impl.go @@ -0,0 +1,79 @@ +package dgraph + +import ( + "context" + "e621_to_neo4j/database" + "e621_to_neo4j/e621/models" + "github.com/dgraph-io/dgo/v210" + "github.com/dgraph-io/dgo/v210/protos/api" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/encoding/gzip" + "log" +) + +type dgraphConnection struct { + driver dgo.Dgraph +} + +func (d *dgraphConnection) UploadUser(ctx context.Context, user models.E621User) error { + //TODO implement me + panic("implement me") +} + +func (d *dgraphConnection) UploadSource(ctx context.Context, SourceURL string) error { + //TODO implement me + panic("implement me") +} + +func (d *dgraphConnection) UploadPost(ctx context.Context, e621ID int64) error { + //TODO implement me + panic("implement me") +} + +func (d *dgraphConnection) UploadTag(ctx context.Context, name string, tagType string) error { + //TODO implement me + panic("implement me") +} + +func (d *dgraphConnection) EstablishPostTagLink(ctx context.Context, e621PostID int64, e621Tag string) error { + //TODO implement me + panic("implement me") +} + +func (d *dgraphConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error { + //TODO implement me + panic("implement me") +} + +func (d *dgraphConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error { + //TODO implement me + panic("implement me") +} + +func (d *dgraphConnection) Connect(ctx context.Context, endpoint string, username string, password string) error { + // Dial a gRPC connection. The address to dial to can be configured when + // setting up the dgraph cluster. + dialOpts := append([]grpc.DialOption{}, + grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions(grpc.UseCompressor(gzip.Name))) + conn, err := grpc.Dial(endpoint, dialOpts...) + + if err != nil { + log.Fatal(err) + } + + defer func(conn *grpc.ClientConn) { + err := conn.Close() + if err != nil { + return + } + }(conn) + + d.driver = *dgo.NewDgraphClient(api.NewDgraphClient(conn)) + return nil +} + +func NewDGraphConnection() database.GraphConnection { + return &dgraphConnection{} +} diff --git a/database/logic.go b/database/logic.go new file mode 100644 index 0000000..6d75eb7 --- /dev/null +++ b/database/logic.go @@ -0,0 +1,27 @@ +package database + +import ( + "context" + "e621_to_neo4j/e621/models" +) + +// [X:Neo4J] Connection +// [X:Neo4J] Upload User +// [X:Neo4J] Upload Source +// [X:Neo4J] Upload Post +// [X:Neo4J] Upload Tag +// [X:Neo4J] Relationship Post->Tag +// [X:Neo4J] Relationship Post->Source +// [X:Neo4J] Relationship User->Post + +type GraphConnection interface { + Connect(ctx context.Context, endpoint string, username string, password string) error + UploadUser(ctx context.Context, user models.E621User) error + UploadSource(ctx context.Context, SourceURL string) error + UploadPost(ctx context.Context, e621ID int64) error + UploadTag(ctx context.Context, name string, tagType string) error + EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error + EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error + EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error + CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error) +} diff --git a/database/neo4j/impl.go b/database/neo4j/impl.go new file mode 100644 index 0000000..ed587f2 --- /dev/null +++ b/database/neo4j/impl.go @@ -0,0 +1,68 @@ +package neo4j + +import ( + "context" + "e621_to_neo4j/database" + "e621_to_neo4j/e621/models" + "github.com/neo4j/neo4j-go-driver/v5/neo4j" + "github.com/neo4j/neo4j-go-driver/v5/neo4j/config" +) + +type neo4jConnection struct { + driver neo4j.DriverWithContext +} + +func (c *neo4jConnection) CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error) { + return RelationshipCheckUserToPost(ctx, c.driver, e621PostID, e621UserID) +} + +func (c *neo4jConnection) EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error { + return EstablishPostTagLink(ctx, c.driver, e621PostID, e621Tag) +} + +func (c *neo4jConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error { + return EstablishPostToSourceLink(ctx, c.driver, e621PostID, sourceURL) +} + +func (c *neo4jConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error { + return EstablishUserToPostLink(ctx, c.driver, e621PostID, e621UserID) +} + +func (c *neo4jConnection) UploadTag(ctx context.Context, name string, tagType string) error { + return CreateTagNode(ctx, c.driver, name, tagType) +} + +func (c *neo4jConnection) UploadPost(ctx context.Context, e621ID int64) error { + return CreatePostNode(ctx, c.driver, e621ID) +} + +func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) error { + return CreateSourceNode(ctx, c.driver, SourceURL) +} + +func (c *neo4jConnection) UploadUser(ctx context.Context, user models.E621User) error { + return CreateUserNode(ctx, c.driver, user) +} +func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error { + driver, err := neo4j.NewDriverWithContext(endpoint, neo4j.BasicAuth(username, password, ""), + useConsoleLogger(neo4j.INFO)) + if err != nil { + return err + } + err = driver.VerifyAuthentication(context.Background(), nil) + if err != nil { + return err + } + c.driver = driver + return nil +} + +func NewNeo4JConnection() database.GraphConnection { + return &neo4jConnection{} +} + +func useConsoleLogger(level neo4j.LogLevel) func(config *config.Config) { + return func(config *config.Config) { + config.Log = neo4j.ConsoleLogger(level) + } +} diff --git a/neo4jAPI/models/tag.go b/database/neo4j/models/tag.go similarity index 100% rename from neo4jAPI/models/tag.go rename to database/neo4j/models/tag.go diff --git a/neo4jAPI/post.go b/database/neo4j/post.go similarity index 96% rename from neo4jAPI/post.go rename to database/neo4j/post.go index 1e7d817..b1056cc 100644 --- a/neo4jAPI/post.go +++ b/database/neo4j/post.go @@ -1,4 +1,4 @@ -package neo4jAPI +package neo4j import ( "context" diff --git a/neo4jAPI/relationship.go b/database/neo4j/relationship.go similarity index 80% rename from neo4jAPI/relationship.go rename to database/neo4j/relationship.go index 5b3ee1c..0094fc3 100644 --- a/neo4jAPI/relationship.go +++ b/database/neo4j/relationship.go @@ -1,11 +1,11 @@ -package neo4jAPI +package neo4j import ( "context" "github.com/neo4j/neo4j-go-driver/v5/neo4j" ) -func PostToTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error { +func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error { query := ` MATCH (p:e621Post {e621PostID: $e621PostID}) MATCH (t:e621Tag {e621Tag: $e621Tag}) @@ -24,7 +24,7 @@ func PostToTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, return nil } -func PostToSourceRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error { +func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error { query := ` MATCH (p:e621Post {e621PostID: $e621PostID}) MATCH (s:Source {URL: $sourceURL}) @@ -43,7 +43,7 @@ func PostToSourceRelationship(ctx context.Context, driver neo4j.DriverWithContex return nil } -func UserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error { +func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error { query := ` MATCH (p:e621Post {e621PostID: $e621PostID}) MATCH (u:e621User {e621ID: $e621ID}) @@ -63,7 +63,7 @@ func UserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext, } // CheckUserToPostRelationship gives back a bool if the connection between the post and the user exists -func CheckUserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) { +func RelationshipCheckUserToPost(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) { query := ` MATCH (user:e621User {e621ID: $e621ID})-[favorite:IS_FAVORITE]->(post:e621Post {e621PostID: $e621PostID}) RETURN COUNT(favorite) > 0 AS isFavorite diff --git a/neo4jAPI/source.go b/database/neo4j/source.go similarity index 95% rename from neo4jAPI/source.go rename to database/neo4j/source.go index b6a6cf3..bbddac2 100644 --- a/neo4jAPI/source.go +++ b/database/neo4j/source.go @@ -1,4 +1,4 @@ -package neo4jAPI +package neo4j import ( "context" diff --git a/neo4jAPI/tag.go b/database/neo4j/tag.go similarity index 51% rename from neo4jAPI/tag.go rename to database/neo4j/tag.go index 8b5fa34..a2c08a6 100644 --- a/neo4jAPI/tag.go +++ b/database/neo4j/tag.go @@ -1,8 +1,8 @@ -package neo4jAPI +package neo4j import ( "context" - "e621_to_neo4j/neo4jAPI/models" + "e621_to_neo4j/database/neo4j/models" "github.com/neo4j/neo4j-go-driver/v5/neo4j" ) @@ -24,55 +24,6 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str return nil } -func CreateTagNode2(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error { - session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite}) - defer session.Close(ctx) - - query := ` - MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType}) - RETURN u -` - params := map[string]interface{}{ - "name": name, - "tagType": tagType, - } - - _, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (any, error) { - result, err := tx.Run(ctx, query, params) - if err != nil { - return nil, err - } - - return result.Consume(ctx) - }) - - return err -} - -func GetAllTagNodes(ctx context.Context, driver neo4j.DriverWithContext, tagType string) ([]string, error) { - query := ` - MATCH (u:e621Tag {e621TagType: $tagType}) - RETURN u.e621Tag as eTag - ` - - params := map[string]interface{}{ - "tagType": tagType, - } - - result, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer) - if err != nil { - return nil, err - } - - var tags []string - for _, record := range result.Records { - tag, _, _ := neo4j.GetRecordValue[string](record, "eTag") - tags = append(tags, tag) - } - - return tags, nil -} - func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string) (models.DBTag, bool, error) { var tag models.DBTag diff --git a/neo4jAPI/user.go b/database/neo4j/user.go similarity index 97% rename from neo4jAPI/user.go rename to database/neo4j/user.go index c5b223a..cb8ec65 100644 --- a/neo4jAPI/user.go +++ b/database/neo4j/user.go @@ -1,4 +1,4 @@ -package neo4jAPI +package neo4j import ( "context" diff --git a/docker-compose.dgraph.yml b/docker-compose.dgraph.yml index 6b1b586..cd1c1d8 100644 --- a/docker-compose.dgraph.yml +++ b/docker-compose.dgraph.yml @@ -2,8 +2,6 @@ version: "3.2" services: zero: image: dgraph/dgraph:latest - volumes: - - /tmp/data:/dgraph ports: - 5080:5080 - 6080:6080 @@ -11,8 +9,6 @@ services: command: dgraph zero --my=zero:5080 alpha: image: dgraph/dgraph:latest - volumes: - - /tmp/data:/dgraph ports: - 8080:8080 - 9080:9080 diff --git a/e621/client.go b/e621/client.go index 1558eed..8f647f6 100644 --- a/e621/client.go +++ b/e621/client.go @@ -1,6 +1,7 @@ package e621 import ( + "e621_to_neo4j/utils" "golang.org/x/time/rate" "net/http" ) @@ -15,6 +16,7 @@ type Client struct { username string client *http.Client limiter *rate.Limiter + queue *utils.Queue } // NewClient creates a new e621 API client. @@ -24,5 +26,6 @@ func NewClient(apiKey string, username string) *Client { username: username, client: &http.Client{}, limiter: rate.NewLimiter(1, 2), + queue: &utils.Queue{}, } } diff --git a/e621/favorite.go b/e621/favorite.go index 6083457..a3bcf38 100644 --- a/e621/favorite.go +++ b/e621/favorite.go @@ -4,19 +4,17 @@ import ( "e621_to_neo4j/e621/models" "fmt" "log" - "time" ) // GetFavorites retrieves all favorites from the e621 API. func (c *Client) GetFavorites(user models.E621User) ([]models.Post, error) { - time.Sleep(1 * time.Second) - var page int64 + page := 1 var allFavorites []models.Post var URIPath string for { URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page) - favorite, err := ExecuteGetAPIRequest[models.APIe621Post](c, URIPath) + favorite, err := ExecuteGetAPIRequest[models.PostResponseWrapper](c, URIPath) if err != nil { log.Printf(err.Error()) } diff --git a/e621/models/post.go b/e621/models/post.go index 0f16df1..dcc7c3f 100644 --- a/e621/models/post.go +++ b/e621/models/post.go @@ -2,17 +2,17 @@ package models import "encoding/json" -func UnmarshalE621Post(data []byte) (APIe621Post, error) { - var r APIe621Post +func UnmarshalE621Post(data []byte) (PostResponseWrapper, error) { + var r PostResponseWrapper err := json.Unmarshal(data, &r) return r, err } -func (r *APIe621Post) Marshal() ([]byte, error) { +func (r *PostResponseWrapper) Marshal() ([]byte, error) { return json.Marshal(r) } -type APIe621Post struct { +type PostResponseWrapper struct { Posts []Post `json:"posts"` } diff --git a/e621/request.go b/e621/request.go index c43205b..38e72f9 100644 --- a/e621/request.go +++ b/e621/request.go @@ -5,12 +5,14 @@ import ( "encoding/json" "fmt" "io" + "log" "net/http" ) func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) { var err error c.limiter.Wait(context.Background()) + log.Println(URIPath) url := fmt.Sprintf("%s/%s", baseURL, URIPath) req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/go.mod b/go.mod index 3f4653e..e6a4f4a 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,18 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/stretchr/testify v1.5.1 // indirect + golang.org/x/net v0.11.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.10.0 // indirect golang.org/x/time v0.3.0 // indirect + google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/grpc v1.56.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 64d991e..599c3f1 100644 --- a/go.sum +++ b/go.sum @@ -3,16 +3,71 @@ github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2Uo github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d h1:abDbP7XBVgwda+h0J5Qra5p2OQpidU2FdkXvzCKL+H8= +github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d/go.mod h1:wKFzULXAPj3U2BDAPWXhSbQQNC6FU1+1/5iika6IY7g= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/neo4j/neo4j-go-driver/v5 v5.8.1 h1:IysKg6KJIUgyItmnHRRrt2N8srbd6znMslRW3qQErTQ= github.com/neo4j/neo4j-go-driver/v5 v5.8.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU= +golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58= +golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 h1:x1vNwUhVOcsYoKyEGCZBH694SBmmBjA2EfauFVEI2+M= +google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE= +google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index afd552b..9fdd2d4 100644 --- a/main.go +++ b/main.go @@ -3,15 +3,18 @@ package main import ( "context" "e621_to_neo4j/api" + "e621_to_neo4j/database" + neo4j "e621_to_neo4j/database/neo4j" "e621_to_neo4j/e621" - "e621_to_neo4j/neo4jAPI" "e621_to_neo4j/utils" - "github.com/neo4j/neo4j-go-driver/v5/neo4j" "log" "net/http" + "strings" ) func main() { + var graphConnection database.GraphConnection + ctx := context.Background() // Loads Config config, err := utils.LoadConfig() @@ -19,29 +22,18 @@ func main() { log.Println(err) } - log.Println("Setup Neo4J Connection") - // Connect to Neo4j DB - driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword) - if err != nil { - log.Fatal(err) - } - - err = driver.VerifyAuthentication(context.Background(), nil) - - if err != nil { - log.Println("Cannot connect and authenticate to NEO4J DB") - log.Fatal(err) - } - - log.Println("Connection successful") - - ctx := context.Background() - defer func(driver neo4j.DriverWithContext, ctx context.Context) { - err := driver.Close(ctx) + switch strings.ToLower(config.DBType) { + case "neo4j": + log.Println("Setup Neo4J Connection") + graphConnection = neo4j.NewNeo4JConnection() + err = graphConnection.Connect(ctx, config.DBEndpoint, config.Neo4jUsername, config.Neo4jPassword) if err != nil { - log.Fatal(err) + panic(err) } - }(driver, ctx) + log.Println("Connection successful") + default: + panic("No Database was selected!") + } // Initialize the e621API e621Client := e621.NewClient(config.E621APIKey, config.E621Username) @@ -49,7 +41,7 @@ func main() { log.Printf("Im ready!") // Register the UserHandler with the "/user" route - http.HandleFunc("/user", api.UserHandler(ctx, driver, e621Client)) + http.HandleFunc("/user", api.UserHandler(ctx, graphConnection, e621Client)) // Start the HTTP server err = http.ListenAndServe(":8080", nil) diff --git a/neo4jAPI/connection.go b/neo4jAPI/connection.go deleted file mode 100644 index 720cc65..0000000 --- a/neo4jAPI/connection.go +++ /dev/null @@ -1,23 +0,0 @@ -package neo4jAPI - -import ( - "fmt" - "github.com/neo4j/neo4j-go-driver/v5/neo4j" - "github.com/neo4j/neo4j-go-driver/v5/neo4j/config" -) - -func NewConnection(uri string, username string, password string) (neo4j.DriverWithContext, error) { - driver, err := neo4j.NewDriverWithContext(uri, neo4j.BasicAuth(username, password, ""), - useConsoleLogger(neo4j.INFO)) - if err != nil { - return nil, fmt.Errorf("failed to create Neo4j driver: %v", err) - } - return driver, nil - -} - -func useConsoleLogger(level neo4j.LogLevel) func(config *config.Config) { - return func(config *config.Config) { - config.Log = neo4j.ConsoleLogger(level) - } -} diff --git a/services/user.go b/services/user.go index 078e634..e45c797 100644 --- a/services/user.go +++ b/services/user.go @@ -2,16 +2,15 @@ package services import ( "context" + "e621_to_neo4j/database" "e621_to_neo4j/e621" "e621_to_neo4j/e621/models" - "e621_to_neo4j/neo4jAPI" "e621_to_neo4j/utils" - "github.com/neo4j/neo4j-go-driver/v5/neo4j" "log" "time" ) -func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error { +func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e621Client e621.Client, username string) error { var err error e621User, err := e621Client.GetUserInfo(username) @@ -26,7 +25,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID) - err = neo4jAPI.CreateUserNode(ctx, driver, e621User) + err = graphConnection.UploadUser(ctx, e621User) if err != nil { log.Fatal(err) } @@ -43,8 +42,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client // Uploads all Tags, Posts as Nodes to Neo4j for i, post := range userFavorites { - - if exists, err := neo4jAPI.CheckUserToPostRelationship(ctx, driver, post.ID, e621User.ID); err == nil && exists { + if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists { log.Printf("No new posts found for user %s with id %d", e621User.Name, e621User.ID) log.Printf("Last Post ID Found: %d", post.ID) break @@ -53,44 +51,44 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client } start = time.Now() - err = uploadNodes(ctx, driver, post) + err = uploadNodes(ctx, graphConnection, post) if err != nil { return err } log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start)) start := time.Now() - err = uploadPostToUserRelationship(ctx, driver, post, e621User) + err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User) if err != nil { log.Fatal(err) return err } - err = uploadSourceTagRelationship(ctx, driver, post) + err = uploadSourceTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } - err = uploadGeneralTagRelationship(ctx, driver, post) + err = uploadGeneralTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } - err = uploadCharacterTagtRelationship(ctx, driver, post) + err = uploadCharacterTagtRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } - err = uploadCopyrightTagRelationship(ctx, driver, post) + err = uploadCopyrightTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } - err = uploadArtistTagRelationship(ctx, driver, post) + err = uploadArtistTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err @@ -103,7 +101,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client } // uploadNodes uploads the post to the database and creates the nodes -func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { +func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { uniqueGeneralTags := make([]string, 0) uniqueCharacterTags := make([]string, 0) @@ -125,41 +123,41 @@ func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post model uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) - err := neo4jAPI.CreatePostNode(ctx, driver, post.ID) + err := graphConnection.UploadPost(ctx, post.ID) if err != nil { return err } // Uploads the source to the database for _, source := range post.Sources { - err := neo4jAPI.CreateSourceNode(ctx, driver, source) + err := graphConnection.UploadSource(ctx, source) if err != nil { return err } } for _, uniqueGeneralTag := range uniqueGeneralTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") + err := graphConnection.UploadTag(ctx, uniqueGeneralTag, "general") if err != nil { return err } } for _, uniqueCharacterTag := range uniqueCharacterTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") + err := graphConnection.UploadTag(ctx, uniqueCharacterTag, "character") if err != nil { return err } } for _, uniqueCopyrightTag := range uniqueCopyrightTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") + err := graphConnection.UploadTag(ctx, uniqueCopyrightTag, "copyright") if err != nil { return err } } for _, uniqueArtistTag := range uniqueArtistTags { - err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") + err := graphConnection.UploadTag(ctx, uniqueArtistTag, "artist") if err != nil { return err } @@ -169,8 +167,8 @@ func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post model } // uploadPostToUserRelationship creates a relationship between the user and the post -func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post, e621User models.E621User) error { - err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID) +func uploadPostToUserRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post, e621User models.E621User) error { + err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID) if err != nil { return err } @@ -179,9 +177,9 @@ func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithCo } // uploadSourceTagRelationship creates a relationship between the post and the source -func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { +func uploadSourceTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { for _, source := range post.Sources { - err := neo4jAPI.PostToSourceRelationship(ctx, driver, post.ID, source) + err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source) if err != nil { return err } @@ -192,9 +190,9 @@ func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithCon } // uploadGeneralTagRelationship creates a relationship between the post and the general tag -func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { +func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { for _, generalTag := range post.Tags.General { - err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, generalTag) + err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag) if err != nil { return err } @@ -205,9 +203,9 @@ func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithCo } // uploadCharacterTagtRelationship creates a relationship between the post and the character tag -func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { +func uploadCharacterTagtRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { for _, characterTag := range post.Tags.Character { - err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, characterTag) + err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag) if err != nil { return err } @@ -218,9 +216,9 @@ func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWit } // uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag -func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { +func uploadCopyrightTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { for _, copyrightTag := range post.Tags.Copyright { - err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, copyrightTag) + err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag) if err != nil { return err } @@ -231,9 +229,9 @@ func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWith } // uploadArtistTagRelationship creates a relationship between the post and the artist tag -func uploadArtistTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error { +func uploadArtistTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { for _, artistTag := range post.Tags.Artist { - err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, artistTag) + err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag) if err != nil { return err } diff --git a/utils/config.go b/utils/config.go index 7f78f7d..117d672 100644 --- a/utils/config.go +++ b/utils/config.go @@ -8,7 +8,8 @@ import ( type Config struct { E621APIKey string `env:"E621_API_KEY,required"` E621Username string `env:"E621_USERNAME,required"` - Neo4jURL string `env:"NEO4J_URL,required"` + DBType string `env:"DB_TYPE,required"` + DBEndpoint string `env:"DB_URL,required"` Neo4jUsername string `env:"NEO4J_USERNAME,required"` Neo4jPassword string `env:"NEO4J_PASSWORD,required"` }