anthrove/e621-sdk-go
Archived
4
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases Wiki Activity

added DB Interface for later dgraph DB backend

Browse Source
This commit is contained in:
David Janowski 2023-06-20 10:38:36 +02:00
parent ce9c081ae6
commit f95e9b0853
21 changed files with 312 additions and 154 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
api/user.go
View File

@ -2,15 +2,15 @@ package api
import (
"context"
"e621_to_neo4j/database"
"e621_to_neo4j/e621"
"e621_to_neo4j/services"
"fmt"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"net/http"
)
// UserHandler is the handler for the user API
func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
func UserHandler(ctx context.Context, graphConnection database.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
@ -26,7 +26,7 @@ func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client
}
// Perform further processing with the username
go services.ScrapeUser(ctx, driver, *e621Client, username)
go services.ScrapeUser(ctx, graphConnection, *e621Client, username)
// Send a response
w.WriteHeader(http.StatusOK)

79
database/dgraph/impl.go Normal file
View File

@ -0,0 +1,79 @@
package dgraph
import (
"context"
"e621_to_neo4j/database"
"e621_to_neo4j/e621/models"
"github.com/dgraph-io/dgo/v210"
"github.com/dgraph-io/dgo/v210/protos/api"
"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/encoding/gzip"
"log"
)
type dgraphConnection struct {
driver dgo.Dgraph
}
func (d *dgraphConnection) UploadUser(ctx context.Context, user models.E621User) error {
//TODO implement me
panic("implement me")
}
func (d *dgraphConnection) UploadSource(ctx context.Context, SourceURL string) error {
//TODO implement me
panic("implement me")
}
func (d *dgraphConnection) UploadPost(ctx context.Context, e621ID int64) error {
//TODO implement me
panic("implement me")
}
func (d *dgraphConnection) UploadTag(ctx context.Context, name string, tagType string) error {
//TODO implement me
panic("implement me")
}
func (d *dgraphConnection) EstablishPostTagLink(ctx context.Context, e621PostID int64, e621Tag string) error {
//TODO implement me
panic("implement me")
}
func (d *dgraphConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error {
//TODO implement me
panic("implement me")
}
func (d *dgraphConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error {
//TODO implement me
panic("implement me")
}
func (d *dgraphConnection) Connect(ctx context.Context, endpoint string, username string, password string) error {
// Dial a gRPC connection. The address to dial to can be configured when
// setting up the dgraph cluster.
dialOpts := append([]grpc.DialOption{},
grpc.WithTransportCredentials(insecure.NewCredentials()),
grpc.WithDefaultCallOptions(grpc.UseCompressor(gzip.Name)))
conn, err := grpc.Dial(endpoint, dialOpts...)
if err != nil {
log.Fatal(err)
}
defer func(conn *grpc.ClientConn) {
err := conn.Close()
if err != nil {
return
}
}(conn)
d.driver = *dgo.NewDgraphClient(api.NewDgraphClient(conn))
return nil
}
func NewDGraphConnection() database.GraphConnection {
return &dgraphConnection{}
}

27
database/logic.go Normal file
View File

@ -0,0 +1,27 @@
package database
import (
"context"
"e621_to_neo4j/e621/models"
)
// [X:Neo4J] Connection
// [X:Neo4J] Upload User
// [X:Neo4J] Upload Source
// [X:Neo4J] Upload Post
// [X:Neo4J] Upload Tag
// [X:Neo4J] Relationship Post->Tag
// [X:Neo4J] Relationship Post->Source
// [X:Neo4J] Relationship User->Post
type GraphConnection interface {
Connect(ctx context.Context, endpoint string, username string, password string) error
UploadUser(ctx context.Context, user models.E621User) error
UploadSource(ctx context.Context, SourceURL string) error
UploadPost(ctx context.Context, e621ID int64) error
UploadTag(ctx context.Context, name string, tagType string) error
EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error
EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error
EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error
CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error)
}

68
database/neo4j/impl.go Normal file
View File

@ -0,0 +1,68 @@
package neo4j
import (
"context"
"e621_to_neo4j/database"
"e621_to_neo4j/e621/models"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"github.com/neo4j/neo4j-go-driver/v5/neo4j/config"
)
type neo4jConnection struct {
driver neo4j.DriverWithContext
}
func (c *neo4jConnection) CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error) {
return RelationshipCheckUserToPost(ctx, c.driver, e621PostID, e621UserID)
}
func (c *neo4jConnection) EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error {
return EstablishPostTagLink(ctx, c.driver, e621PostID, e621Tag)
}
func (c *neo4jConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error {
return EstablishPostToSourceLink(ctx, c.driver, e621PostID, sourceURL)
}
func (c *neo4jConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error {
return EstablishUserToPostLink(ctx, c.driver, e621PostID, e621UserID)
}
func (c *neo4jConnection) UploadTag(ctx context.Context, name string, tagType string) error {
return CreateTagNode(ctx, c.driver, name, tagType)
}
func (c *neo4jConnection) UploadPost(ctx context.Context, e621ID int64) error {
return CreatePostNode(ctx, c.driver, e621ID)
}
func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) error {
return CreateSourceNode(ctx, c.driver, SourceURL)
}
func (c *neo4jConnection) UploadUser(ctx context.Context, user models.E621User) error {
return CreateUserNode(ctx, c.driver, user)
}
func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error {
driver, err := neo4j.NewDriverWithContext(endpoint, neo4j.BasicAuth(username, password, ""),
useConsoleLogger(neo4j.INFO))
if err != nil {
return err
}
err = driver.VerifyAuthentication(context.Background(), nil)
if err != nil {
return err
}
c.driver = driver
return nil
}
func NewNeo4JConnection() database.GraphConnection {
return &neo4jConnection{}
}
func useConsoleLogger(level neo4j.LogLevel) func(config *config.Config) {
return func(config *config.Config) {
config.Log = neo4j.ConsoleLogger(level)
}
}

0
neo4jAPI/models/tag.go → database/neo4j/models/tag.go
View File

2
neo4jAPI/post.go → database/neo4j/post.go
View File

@ -1,4 +1,4 @@
package neo4jAPI
package neo4j
import (
"context"

10
neo4jAPI/relationship.go → database/neo4j/relationship.go
View File

@ -1,11 +1,11 @@
package neo4jAPI
package neo4j
import (
"context"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
)
func PostToTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error {
func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error {
query := `
MATCH (p:e621Post {e621PostID: $e621PostID})
MATCH (t:e621Tag {e621Tag: $e621Tag})
@ -24,7 +24,7 @@ func PostToTagRelationship(ctx context.Context, driver neo4j.DriverWithContext,
return nil
}
func PostToSourceRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error {
func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error {
query := `
MATCH (p:e621Post {e621PostID: $e621PostID})
MATCH (s:Source {URL: $sourceURL})
@ -43,7 +43,7 @@ func PostToSourceRelationship(ctx context.Context, driver neo4j.DriverWithContex
return nil
}
func UserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error {
func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error {
query := `
MATCH (p:e621Post {e621PostID: $e621PostID})
MATCH (u:e621User {e621ID: $e621ID})
@ -63,7 +63,7 @@ func UserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext,
}
// CheckUserToPostRelationship gives back a bool if the connection between the post and the user exists
func CheckUserToPostRelationship(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) {
func RelationshipCheckUserToPost(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) {
query := `
MATCH (user:e621User {e621ID: $e621ID})-[favorite:IS_FAVORITE]->(post:e621Post {e621PostID: $e621PostID})
RETURN COUNT(favorite) > 0 AS isFavorite

2
neo4jAPI/source.go → database/neo4j/source.go
View File

@ -1,4 +1,4 @@
package neo4jAPI
package neo4j
import (
"context"

53
neo4jAPI/tag.go → database/neo4j/tag.go
View File

@ -1,8 +1,8 @@
package neo4jAPI
package neo4j
import (
"context"
"e621_to_neo4j/neo4jAPI/models"
"e621_to_neo4j/database/neo4j/models"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
)
@ -24,55 +24,6 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str
return nil
}
func CreateTagNode2(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error {
session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close(ctx)
query := `
MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType})
RETURN u
`
params := map[string]interface{}{
"name": name,
"tagType": tagType,
}
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (any, error) {
result, err := tx.Run(ctx, query, params)
if err != nil {
return nil, err
}
return result.Consume(ctx)
})
return err
}
func GetAllTagNodes(ctx context.Context, driver neo4j.DriverWithContext, tagType string) ([]string, error) {
query := `
MATCH (u:e621Tag {e621TagType: $tagType})
RETURN u.e621Tag as eTag
`
params := map[string]interface{}{
"tagType": tagType,
}
result, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)
if err != nil {
return nil, err
}
var tags []string
for _, record := range result.Records {
tag, _, _ := neo4j.GetRecordValue[string](record, "eTag")
tags = append(tags, tag)
}
return tags, nil
}
func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string) (models.DBTag, bool, error) {
var tag models.DBTag

2
neo4jAPI/user.go → database/neo4j/user.go
View File

@ -1,4 +1,4 @@
package neo4jAPI
package neo4j
import (
"context"

4
docker-compose.dgraph.yml
View File

@ -2,8 +2,6 @@ version: "3.2"
services:
zero:
image: dgraph/dgraph:latest
volumes:
- /tmp/data:/dgraph
ports:
- 5080:5080
- 6080:6080
@ -11,8 +9,6 @@ services:
command: dgraph zero --my=zero:5080
alpha:
image: dgraph/dgraph:latest
volumes:
- /tmp/data:/dgraph
ports:
- 8080:8080
- 9080:9080

3
e621/client.go
View File

@ -1,6 +1,7 @@
package e621
import (
"e621_to_neo4j/utils"
"golang.org/x/time/rate"
"net/http"
)
@ -15,6 +16,7 @@ type Client struct {
username string
client *http.Client
limiter *rate.Limiter
queue *utils.Queue
}
// NewClient creates a new e621 API client.
@ -24,5 +26,6 @@ func NewClient(apiKey string, username string) *Client {
username: username,
client: &http.Client{},
limiter: rate.NewLimiter(1, 2),
queue: &utils.Queue{},
}
}

6
e621/favorite.go
View File

@ -4,19 +4,17 @@ import (
"e621_to_neo4j/e621/models"
"fmt"
"log"
"time"
)
// GetFavorites retrieves all favorites from the e621 API.
func (c *Client) GetFavorites(user models.E621User) ([]models.Post, error) {
time.Sleep(1 * time.Second)
var page int64
page := 1
var allFavorites []models.Post
var URIPath string
for {
URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page)
favorite, err := ExecuteGetAPIRequest[models.APIe621Post](c, URIPath)
favorite, err := ExecuteGetAPIRequest[models.PostResponseWrapper](c, URIPath)
if err != nil {
log.Printf(err.Error())
}

8
e621/models/post.go
View File

@ -2,17 +2,17 @@ package models
import "encoding/json"
func UnmarshalE621Post(data []byte) (APIe621Post, error) {
var r APIe621Post
func UnmarshalE621Post(data []byte) (PostResponseWrapper, error) {
var r PostResponseWrapper
err := json.Unmarshal(data, &r)
return r, err
}
func (r *APIe621Post) Marshal() ([]byte, error) {
func (r *PostResponseWrapper) Marshal() ([]byte, error) {
return json.Marshal(r)
}
type APIe621Post struct {
type PostResponseWrapper struct {
Posts []Post `json:"posts"`
}

2
e621/request.go
View File

@ -5,12 +5,14 @@ import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
)
func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) {
var err error
c.limiter.Wait(context.Background())
log.Println(URIPath)
url := fmt.Sprintf("%s/%s", baseURL, URIPath)
req, err := http.NewRequest("GET", url, nil)
if err != nil {

11
go.mod
View File

@ -9,7 +9,18 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/stretchr/testify v1.5.1 // indirect
golang.org/x/net v0.11.0 // indirect
golang.org/x/sys v0.9.0 // indirect
golang.org/x/text v0.10.0 // indirect
golang.org/x/time v0.3.0 // indirect
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
google.golang.org/grpc v1.56.0 // indirect
google.golang.org/protobuf v1.30.0 // indirect
gopkg.in/yaml.v2 v2.4.0 // indirect
)

55
go.sum
View File

@ -3,16 +3,71 @@ github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2Uo
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d h1:abDbP7XBVgwda+h0J5Qra5p2OQpidU2FdkXvzCKL+H8=
github.com/dgraph-io/dgo/v210 v210.0.0-20230328113526-b66f8ae53a2d/go.mod h1:wKFzULXAPj3U2BDAPWXhSbQQNC6FU1+1/5iika6IY7g=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/neo4j/neo4j-go-driver/v5 v5.8.1 h1:IysKg6KJIUgyItmnHRRrt2N8srbd6znMslRW3qQErTQ=
github.com/neo4j/neo4j-go-driver/v5 v5.8.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.11.0 h1:Gi2tvZIJyBtO9SDr1q9h5hEQCp/4L2RQ+ar0qjx2oNU=
golang.org/x/net v0.11.0/go.mod h1:2L/ixqYpgIVXmeoSA/4Lu7BzTG4KIyPIryS4IsOd1oQ=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s=
golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.10.0 h1:UpjohKhiEgNc0CSauXmwYftY1+LlaC75SJwh0SgCX58=
golang.org/x/text v0.10.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4=
golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0 h1:x1vNwUhVOcsYoKyEGCZBH694SBmmBjA2EfauFVEI2+M=
google.golang.org/genproto v0.0.0-20230525234025-438c736192d0/go.mod h1:9ExIQyXL5hZrHzQceCwuSYwZZ5QZBazOcprJ5rgs3lY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
google.golang.org/grpc v1.56.0 h1:+y7Bs8rtMd07LeXmL3NxcTLn7mUkbKZqEpPhMNkwJEE=
google.golang.org/grpc v1.56.0/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s=
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng=
google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

34
main.go
View File

@ -3,15 +3,18 @@ package main
import (
"context"
"e621_to_neo4j/api"
"e621_to_neo4j/database"
neo4j "e621_to_neo4j/database/neo4j"
"e621_to_neo4j/e621"
"e621_to_neo4j/neo4jAPI"
"e621_to_neo4j/utils"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"log"
"net/http"
"strings"
)
func main() {
var graphConnection database.GraphConnection
ctx := context.Background()
// Loads Config
config, err := utils.LoadConfig()
@ -19,29 +22,18 @@ func main() {
log.Println(err)
}
switch strings.ToLower(config.DBType) {
case "neo4j":
log.Println("Setup Neo4J Connection")
// Connect to Neo4j DB
driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
graphConnection = neo4j.NewNeo4JConnection()
err = graphConnection.Connect(ctx, config.DBEndpoint, config.Neo4jUsername, config.Neo4jPassword)
if err != nil {
log.Fatal(err)
panic(err)
}
err = driver.VerifyAuthentication(context.Background(), nil)
if err != nil {
log.Println("Cannot connect and authenticate to NEO4J DB")
log.Fatal(err)
}
log.Println("Connection successful")
ctx := context.Background()
defer func(driver neo4j.DriverWithContext, ctx context.Context) {
err := driver.Close(ctx)
if err != nil {
log.Fatal(err)
default:
panic("No Database was selected!")
}
}(driver, ctx)
// Initialize the e621API
e621Client := e621.NewClient(config.E621APIKey, config.E621Username)
@ -49,7 +41,7 @@ func main() {
log.Printf("Im ready!")
// Register the UserHandler with the "/user" route
http.HandleFunc("/user", api.UserHandler(ctx, driver, e621Client))
http.HandleFunc("/user", api.UserHandler(ctx, graphConnection, e621Client))
// Start the HTTP server
err = http.ListenAndServe(":8080", nil)

23
neo4jAPI/connection.go
View File

@ -1,23 +0,0 @@
package neo4jAPI
import (
"fmt"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"github.com/neo4j/neo4j-go-driver/v5/neo4j/config"
)
func NewConnection(uri string, username string, password string) (neo4j.DriverWithContext, error) {
driver, err := neo4j.NewDriverWithContext(uri, neo4j.BasicAuth(username, password, ""),
useConsoleLogger(neo4j.INFO))
if err != nil {
return nil, fmt.Errorf("failed to create Neo4j driver: %v", err)
}
return driver, nil
}
func useConsoleLogger(level neo4j.LogLevel) func(config *config.Config) {
return func(config *config.Config) {
config.Log = neo4j.ConsoleLogger(level)
}
}

62
services/user.go
View File

@ -2,16 +2,15 @@ package services
import (
"context"
"e621_to_neo4j/database"
"e621_to_neo4j/e621"
"e621_to_neo4j/e621/models"
"e621_to_neo4j/neo4jAPI"
"e621_to_neo4j/utils"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"log"
"time"
)
func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error {
func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e621Client e621.Client, username string) error {
var err error
e621User, err := e621Client.GetUserInfo(username)
@ -26,7 +25,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID)
err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
err = graphConnection.UploadUser(ctx, e621User)
if err != nil {
log.Fatal(err)
}
@ -43,8 +42,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
// Uploads all Tags, Posts as Nodes to Neo4j
for i, post := range userFavorites {
if exists, err := neo4jAPI.CheckUserToPostRelationship(ctx, driver, post.ID, e621User.ID); err == nil && exists {
if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists {
log.Printf("No new posts found for user %s with id %d", e621User.Name, e621User.ID)
log.Printf("Last Post ID Found: %d", post.ID)
break
@ -53,44 +51,44 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
}
start = time.Now()
err = uploadNodes(ctx, driver, post)
err = uploadNodes(ctx, graphConnection, post)
if err != nil {
return err
}
log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))
start := time.Now()
err = uploadPostToUserRelationship(ctx, driver, post, e621User)
err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User)
if err != nil {
log.Fatal(err)
return err
}
err = uploadSourceTagRelationship(ctx, driver, post)
err = uploadSourceTagRelationship(ctx, graphConnection, post)
if err != nil {
log.Fatal(err)
return err
}
err = uploadGeneralTagRelationship(ctx, driver, post)
err = uploadGeneralTagRelationship(ctx, graphConnection, post)
if err != nil {
log.Fatal(err)
return err
}
err = uploadCharacterTagtRelationship(ctx, driver, post)
err = uploadCharacterTagtRelationship(ctx, graphConnection, post)
if err != nil {
log.Fatal(err)
return err
}
err = uploadCopyrightTagRelationship(ctx, driver, post)
err = uploadCopyrightTagRelationship(ctx, graphConnection, post)
if err != nil {
log.Fatal(err)
return err
}
err = uploadArtistTagRelationship(ctx, driver, post)
err = uploadArtistTagRelationship(ctx, graphConnection, post)
if err != nil {
log.Fatal(err)
return err
@ -103,7 +101,7 @@ func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client
}
// uploadNodes uploads the post to the database and creates the nodes
func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
uniqueGeneralTags := make([]string, 0)
uniqueCharacterTags := make([]string, 0)
@ -125,41 +123,41 @@ func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post model
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
err := graphConnection.UploadPost(ctx, post.ID)
if err != nil {
return err
}
// Uploads the source to the database
for _, source := range post.Sources {
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
err := graphConnection.UploadSource(ctx, source)
if err != nil {
return err
}
}
for _, uniqueGeneralTag := range uniqueGeneralTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
err := graphConnection.UploadTag(ctx, uniqueGeneralTag, "general")
if err != nil {
return err
}
}
for _, uniqueCharacterTag := range uniqueCharacterTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
err := graphConnection.UploadTag(ctx, uniqueCharacterTag, "character")
if err != nil {
return err
}
}
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
err := graphConnection.UploadTag(ctx, uniqueCopyrightTag, "copyright")
if err != nil {
return err
}
}
for _, uniqueArtistTag := range uniqueArtistTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
err := graphConnection.UploadTag(ctx, uniqueArtistTag, "artist")
if err != nil {
return err
}
@ -169,8 +167,8 @@ func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post model
}
// uploadPostToUserRelationship creates a relationship between the user and the post
func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post, e621User models.E621User) error {
err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
func uploadPostToUserRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post, e621User models.E621User) error {
err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID)
if err != nil {
return err
}
@ -179,9 +177,9 @@ func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithCo
}
// uploadSourceTagRelationship creates a relationship between the post and the source
func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
func uploadSourceTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
for _, source := range post.Sources {
err := neo4jAPI.PostToSourceRelationship(ctx, driver, post.ID, source)
err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source)
if err != nil {
return err
}
@ -192,9 +190,9 @@ func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithCon
}
// uploadGeneralTagRelationship creates a relationship between the post and the general tag
func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
for _, generalTag := range post.Tags.General {
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, generalTag)
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag)
if err != nil {
return err
}
@ -205,9 +203,9 @@ func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithCo
}
// uploadCharacterTagtRelationship creates a relationship between the post and the character tag
func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
func uploadCharacterTagtRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
for _, characterTag := range post.Tags.Character {
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, characterTag)
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag)
if err != nil {
return err
}
@ -218,9 +216,9 @@ func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWit
}
// uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag
func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
func uploadCopyrightTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
for _, copyrightTag := range post.Tags.Copyright {
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, copyrightTag)
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag)
if err != nil {
return err
}
@ -231,9 +229,9 @@ func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWith
}
// uploadArtistTagRelationship creates a relationship between the post and the artist tag
func uploadArtistTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
func uploadArtistTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
for _, artistTag := range post.Tags.Artist {
err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, artistTag)
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag)
if err != nil {
return err
}

3
utils/config.go
View File

@ -8,7 +8,8 @@ import (
type Config struct {
E621APIKey string `env:"E621_API_KEY,required"`
E621Username string `env:"E621_USERNAME,required"`
Neo4jURL string `env:"NEO4J_URL,required"`
DBType string `env:"DB_TYPE,required"`
DBEndpoint string `env:"DB_URL,required"`
Neo4jUsername string `env:"NEO4J_USERNAME,required"`
Neo4jPassword string `env:"NEO4J_PASSWORD,required"`
}