From 60b3502ee384a8dd23a1b40b9b3d599136f8888d Mon Sep 17 00:00:00 2001 From: SoXX Date: Wed, 8 Nov 2023 13:01:27 +0000 Subject: [PATCH] implement_new_e621_sdk_#17 (#18) As mentioned in #17, I implemented the new SDK. I removed the scheduler and executor code since they were no longer needed. Reviewed-on: https://git.dragse.it/anthrove/e621-to-graph/pulls/18 Reviewed-by: Lennard Brinkhaus Reviewed-by: daskadse Co-authored-by: SoXX Co-committed-by: SoXX --- .gitea/workflows/build_check.yaml | 2 +- build/package/Dockerfile | 4 +- cmd/scraper/main.go | 6 +- deployments/docker-compose.yaml | 5 +- go.mod | 11 ++- go.sum | 6 ++ internal/api/user.go | 6 +- internal/database/neo4j/impl.go | 18 ++-- internal/database/neo4j/post.go | 7 +- internal/database/neo4j/relationship.go | 11 +-- internal/database/neo4j/user.go | 4 +- internal/e621/client.go | 24 ------ internal/e621/favorites.go | 22 ----- internal/e621/scheduler/executor.go | 43 ---------- internal/e621/scheduler/scheduler.go | 75 ----------------- internal/e621/scheduler/task.go | 43 ---------- internal/e621/task.go | 70 ---------------- internal/e621/user.go | 19 ----- internal/service/manager.go | 43 +++------- pkg/e621/model/post.go | 102 ----------------------- pkg/e621/model/user.go | 83 ------------------- pkg/e621/task.go | 22 ----- pkg/error/error.go | 105 ------------------------ pkg/logic/database.go | 14 ++-- 24 files changed, 66 insertions(+), 679 deletions(-) delete mode 100644 internal/e621/client.go delete mode 100644 internal/e621/favorites.go delete mode 100644 internal/e621/scheduler/executor.go delete mode 100644 internal/e621/scheduler/scheduler.go delete mode 100644 internal/e621/scheduler/task.go delete mode 100644 internal/e621/task.go delete mode 100644 internal/e621/user.go delete mode 100644 pkg/e621/model/post.go delete mode 100644 pkg/e621/model/user.go delete mode 100644 pkg/e621/task.go delete mode 100644 pkg/error/error.go diff --git a/.gitea/workflows/build_check.yaml b/.gitea/workflows/build_check.yaml index 2bb31e1..dc7cd9c 100644 --- a/.gitea/workflows/build_check.yaml +++ b/.gitea/workflows/build_check.yaml @@ -14,7 +14,7 @@ jobs: uses: https://github.com/actions/setup-go@v3 with: # The Go version to download (if necessary) and use. Supports semver spec and ranges. - go-version: 1.20 # optional + go-version: 1.21.3 # optional # Path to the go.mod file. go-version-file: ./go.mod # optional # Set this option to true if you want the action to always check for the latest available version that satisfies the version spec diff --git a/build/package/Dockerfile b/build/package/Dockerfile index 495124d..a164196 100644 --- a/build/package/Dockerfile +++ b/build/package/Dockerfile @@ -1,6 +1,6 @@ FROM golang:alpine as builder -WORKDIR /go/src/git.dragse.it/fenpaws/e621-to-neo4j +WORKDIR /go/src/git.dragse.it/fenpaws/e621-to-graph RUN apk add -U --no-cache ca-certificates && update-ca-certificates @@ -8,8 +8,8 @@ COPY go.mod go.sum ./ RUN go mod download COPY . ./ -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags "-w -s" -o /app +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags "-w -s" -o /app ./cmd/scraper/ FROM scratch diff --git a/cmd/scraper/main.go b/cmd/scraper/main.go index ebfedb2..b660217 100644 --- a/cmd/scraper/main.go +++ b/cmd/scraper/main.go @@ -2,10 +2,10 @@ package main import ( "context" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621" "git.dragse.it/anthrove/e621-to-graph/internal/api" "git.dragse.it/anthrove/e621-to-graph/internal/config" "git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j" - "git.dragse.it/anthrove/e621-to-graph/internal/e621" "git.dragse.it/anthrove/e621-to-graph/pkg/logic" log "github.com/sirupsen/logrus" "net/http" @@ -42,9 +42,9 @@ func main() { }).Info("main: database connection successful") // Initialize the e621API - e621Client := e621.NewClient(appConfig.E621APIKey, appConfig.E621Username) + client := e621.NewClient(appConfig.E621Username, appConfig.E621APIKey) // Register the ScapeUserFavourites with the "/user" route - http.HandleFunc("/user", api.ScapeUserFavourites(ctx, graphConnection, e621Client)) + http.HandleFunc("/user", api.ScapeUserFavourites(ctx, graphConnection, &client)) // Start the HTTP server err = http.ListenAndServe(":8080", nil) diff --git a/deployments/docker-compose.yaml b/deployments/docker-compose.yaml index c449062..47f4fbb 100644 --- a/deployments/docker-compose.yaml +++ b/deployments/docker-compose.yaml @@ -1,7 +1,10 @@ services: app: restart: unless-stopped - build: . + image: anthrove/e621-to-graph:latest + build: + context: ../. + dockerfile: build/package/Dockerfile ports: - 8080:8080 env_file: .env diff --git a/go.mod b/go.mod index 8dbbecb..0270300 100644 --- a/go.mod +++ b/go.mod @@ -1,12 +1,17 @@ module git.dragse.it/anthrove/e621-to-graph -go 1.20 +go 1.21.3 require ( + git.dragse.it/anthrove/e621-sdk-go v1.3.0 github.com/caarlos0/env v3.5.0+incompatible github.com/neo4j/neo4j-go-driver/v5 v5.8.1 github.com/sirupsen/logrus v1.9.3 - golang.org/x/time v0.3.0 + ) -require golang.org/x/sys v0.9.0 // indirect +require ( + github.com/joho/godotenv v1.5.1 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/time v0.3.0 // indirect +) diff --git a/go.sum b/go.sum index 2a566be..226976d 100644 --- a/go.sum +++ b/go.sum @@ -1,8 +1,14 @@ +git.dragse.it/anthrove/e621-sdk-go v1.3.0 h1:sbjrOps4WxXf42kyNb8ZVxC6dCiFrCkqv4C8BgKkIGA= +git.dragse.it/anthrove/e621-sdk-go v1.3.0/go.mod h1:urFl0jjx2UK8Vz1tMI253CvWK8fZqd/a9+xdE8ZBwq4= github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww= +github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/neo4j/neo4j-go-driver/v5 v5.8.1 h1:IysKg6KJIUgyItmnHRRrt2N8srbd6znMslRW3qQErTQ= github.com/neo4j/neo4j-go-driver/v5 v5.8.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/internal/api/user.go b/internal/api/user.go index 11603ea..d80058c 100644 --- a/internal/api/user.go +++ b/internal/api/user.go @@ -3,7 +3,7 @@ package api import ( "context" "fmt" - "git.dragse.it/anthrove/e621-to-graph/internal/e621" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621" "git.dragse.it/anthrove/e621-to-graph/internal/service" "git.dragse.it/anthrove/e621-to-graph/pkg/logic" log "github.com/sirupsen/logrus" @@ -11,7 +11,7 @@ import ( ) // ScapeUserFavourites is the handler for the user API -func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) { +func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnection, client *e621.Client) func(response http.ResponseWriter, request *http.Request) { return func(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { w.WriteHeader(http.StatusMethodNotAllowed) @@ -27,7 +27,7 @@ func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnect } // Perform further processing with the username - go service.ScrapeUser(ctx, graphConnection, *e621Client, username) + go service.ScrapeUser(ctx, graphConnection, client, username) // Send a response w.WriteHeader(http.StatusOK) diff --git a/internal/database/neo4j/impl.go b/internal/database/neo4j/impl.go index 26c695c..3189ef3 100644 --- a/internal/database/neo4j/impl.go +++ b/internal/database/neo4j/impl.go @@ -2,7 +2,7 @@ package neo4j import ( "context" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model" "git.dragse.it/anthrove/e621-to-graph/pkg/logic" "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/neo4j/neo4j-go-driver/v5/neo4j/config" @@ -20,19 +20,19 @@ func NewNeo4JConnection(neo4jDebug bool) logic.GraphConnection { } } -func (c *neo4jConnection) CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error) { - return RelationshipCheckUserToPost(ctx, c.driver, e621PostID, e621UserID) +func (c *neo4jConnection) CheckUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) (bool, error) { + return CheckUserToPostLink(ctx, c.driver, e621PostID, e621UserID) } -func (c *neo4jConnection) EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error { +func (c *neo4jConnection) EstablishPostToTagLink(ctx context.Context, e621PostID model.PostID, e621Tag string) error { return EstablishPostTagLink(ctx, c.driver, e621PostID, e621Tag) } -func (c *neo4jConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error { +func (c *neo4jConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID model.PostID, sourceURL string) error { return EstablishPostToSourceLink(ctx, c.driver, e621PostID, sourceURL) } -func (c *neo4jConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error { +func (c *neo4jConnection) EstablishUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) error { return EstablishUserToPostLink(ctx, c.driver, e621PostID, e621UserID) } @@ -40,15 +40,15 @@ func (c *neo4jConnection) UploadTag(ctx context.Context, name string, tagType st return CreateTagNode(ctx, c.driver, name, tagType) } -func (c *neo4jConnection) UploadPost(ctx context.Context, e621ID int64) error { - return CreatePostNode(ctx, c.driver, e621ID) +func (c *neo4jConnection) UploadPost(ctx context.Context, postID model.PostID) error { + return CreatePostNode(ctx, c.driver, postID) } func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) error { return CreateSourceNode(ctx, c.driver, SourceURL) } -func (c *neo4jConnection) UploadUser(ctx context.Context, user model.E621User) error { +func (c *neo4jConnection) UploadUser(ctx context.Context, user model.User) error { return CreateUserNode(ctx, c.driver, user) } func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error { diff --git a/internal/database/neo4j/post.go b/internal/database/neo4j/post.go index b1056cc..9cd81aa 100644 --- a/internal/database/neo4j/post.go +++ b/internal/database/neo4j/post.go @@ -2,16 +2,17 @@ package neo4j import ( "context" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model" "github.com/neo4j/neo4j-go-driver/v5/neo4j" ) -func CreatePostNode(ctx context.Context, driver neo4j.DriverWithContext, e621ID int64) error { +func CreatePostNode(ctx context.Context, driver neo4j.DriverWithContext, postID model.PostID) error { query := ` - MERGE (u:e621Post {e621PostID: $e621ID}) + MERGE (u:e621Post {e621PostID: $postID}) RETURN u ` params := map[string]any{ - "e621ID": e621ID, + "postID": postID, } _, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer) diff --git a/internal/database/neo4j/relationship.go b/internal/database/neo4j/relationship.go index 3411ecf..317a0bc 100644 --- a/internal/database/neo4j/relationship.go +++ b/internal/database/neo4j/relationship.go @@ -2,11 +2,12 @@ package neo4j import ( "context" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model" "github.com/neo4j/neo4j-go-driver/v5/neo4j" log "github.com/sirupsen/logrus" ) -func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error { +func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, e621Tag string) error { query := ` MATCH (p:e621Post {e621PostID: $e621PostID}) MATCH (t:e621Tag {e621Tag: $e621Tag}) @@ -29,7 +30,7 @@ func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e return nil } -func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error { +func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, sourceURL string) error { query := ` MATCH (p:e621Post {e621PostID: $e621PostID}) MATCH (s:Source {URL: $sourceURL}) @@ -52,7 +53,7 @@ func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithConte return nil } -func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error { +func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, e621UserID model.UserID) error { query := ` MATCH (p:e621Post {e621PostID: $e621PostID}) MATCH (u:e621User {e621ID: $e621ID}) @@ -74,8 +75,8 @@ func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext return nil } -// CheckUserToPostRelationship gives back a bool if the connection between the post and the user exists -func RelationshipCheckUserToPost(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) { +// CheckUserToPostLink gives back a bool if the connection between the post and the user exists +func CheckUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, e621UserID model.UserID) (bool, error) { query := ` MATCH (user:e621User {e621ID: $e621ID})-[favorite:IS_FAVORITE]->(post:e621Post {e621PostID: $e621PostID}) RETURN COUNT(favorite) > 0 AS isFavorite diff --git a/internal/database/neo4j/user.go b/internal/database/neo4j/user.go index 9541a2a..b57847c 100644 --- a/internal/database/neo4j/user.go +++ b/internal/database/neo4j/user.go @@ -2,11 +2,11 @@ package neo4j import ( "context" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model" "github.com/neo4j/neo4j-go-driver/v5/neo4j" ) -func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user model.E621User) error { +func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user model.User) error { query := ` MERGE (u:e621User {e621ID: $id, e621Username: $name}) RETURN u diff --git a/internal/e621/client.go b/internal/e621/client.go deleted file mode 100644 index b1163d1..0000000 --- a/internal/e621/client.go +++ /dev/null @@ -1,24 +0,0 @@ -package e621 - -import ( - e621 "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler" - "golang.org/x/time/rate" -) - -// Client represents the e621 API client. -type Client struct { - apiKey string - username string - scheduler *e621.Scheduler -} - -// NewClient creates a new e621 API client. -func NewClient(apiKey string, username string) *Client { - scheduler := e621.NewScheduler() - scheduler.SetLimiter(rate.NewLimiter(1, 2)) - return &Client{ - apiKey: apiKey, - username: username, - scheduler: scheduler, - } -} diff --git a/internal/e621/favorites.go b/internal/e621/favorites.go deleted file mode 100644 index 396a31c..0000000 --- a/internal/e621/favorites.go +++ /dev/null @@ -1,22 +0,0 @@ -package e621 - -import ( - "context" - "fmt" - "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" - log "github.com/sirupsen/logrus" -) - -// GetFavorites retrieves all favorites from the e621 API. -func (c *Client) GetFavorites(_ context.Context, user model.E621User, page int64) func() (model.PostResponseWrapper, error) { - URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page) - log.WithFields(log.Fields{ - "user": user.Name, - "id": user.ID, - "page": page, - "uri": URIPath, - }).Trace("e621: adding task to get favorites") - e621Task := NewE621ApiTask[model.PostResponseWrapper](URIPath) - return scheduler.Schedule[model.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey) -} diff --git a/internal/e621/scheduler/executor.go b/internal/e621/scheduler/executor.go deleted file mode 100644 index c6134d5..0000000 --- a/internal/e621/scheduler/executor.go +++ /dev/null @@ -1,43 +0,0 @@ -package scheduler - -import ( - "fmt" - "git.dragse.it/anthrove/e621-to-graph/pkg/util/queue" - log "github.com/sirupsen/logrus" - "net/http" -) - -const ( - BASEURL = "https://e621.net" -) - -func GetAPIRequest(schedulerTask queue.SchedulerTask) { - var err error - log.Debug("executing scheduler task") - - url := fmt.Sprintf("%s/%s", BASEURL, schedulerTask.UriPath()) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - schedulerTask.SendError(err) - return - } - - req.Header.Set("User-Agent", "FavGetter (by Selloo)") - req.Header.Add("Accept", "application/json") - req.SetBasicAuth(schedulerTask.BasicAuth()) - - client := http.Client{} - - resp, err := client.Do(req) - if err != nil { - schedulerTask.SendStatusCode(resp.StatusCode) - return - } - - if resp.StatusCode != http.StatusOK { - schedulerTask.SendStatusCode(resp.StatusCode) - return - } - - schedulerTask.SendResponse(resp) -} diff --git a/internal/e621/scheduler/scheduler.go b/internal/e621/scheduler/scheduler.go deleted file mode 100644 index b12babb..0000000 --- a/internal/e621/scheduler/scheduler.go +++ /dev/null @@ -1,75 +0,0 @@ -package scheduler - -import ( - "context" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621" - "git.dragse.it/anthrove/e621-to-graph/pkg/util/queue" - log "github.com/sirupsen/logrus" - "golang.org/x/time/rate" - "reflect" -) - -type Scheduler struct { - queue queue.Queue - limiter *rate.Limiter -} - -func NewScheduler() *Scheduler { - scheduler := &Scheduler{ - queue: queue.NewQueue(), - limiter: nil, - } - go scheduler.StartExecutionHandler() - return scheduler -} - -func Schedule[T e621.DataType](s *Scheduler, t e621.Task[T], username string, apiKey string) func() (T, error) { - channel := make(chan e621.DataResponse[T]) - schedulerTask := NewSchedulerTaskImpl[T](t, channel, username, apiKey) - log.WithFields(log.Fields{ - "type": reflect.TypeOf(*new(T)), - }).Debug("scheduler: pushing task to queue") - - err := s.queue.Push(schedulerTask) - if err != nil { - return func() (T, error) { - var nil T - return nil, err - } - } - - log.WithFields(log.Fields{ - "type": reflect.TypeOf(*new(T)), - }).Trace("scheduler: pushed task to queue") - - return func() (T, error) { - log.Trace("scheduler: getting data from channel") - data := <-channel - return data.Data, data.Error - } -} - -func (s *Scheduler) SetLimiter(limiter *rate.Limiter) { - log.WithFields(log.Fields{ - "burst": limiter.Burst(), - "limit": limiter.Limit(), - "tokens": limiter.Tokens(), - }).Debug("scheduler: setting limiter") - s.limiter = limiter -} - -func (s *Scheduler) StartExecutionHandler() { - for { - if s.limiter != nil { - s.limiter.Wait(context.Background()) - } - s.queue.WaitForElement() - log.Trace("scheduler: element found") - task, err := s.queue.Pop() - if err != nil { - continue - } - GetAPIRequest(task) - } - -} diff --git a/internal/e621/scheduler/task.go b/internal/e621/scheduler/task.go deleted file mode 100644 index e158740..0000000 --- a/internal/e621/scheduler/task.go +++ /dev/null @@ -1,43 +0,0 @@ -package scheduler - -import ( - "git.dragse.it/anthrove/e621-to-graph/pkg/e621" - "git.dragse.it/anthrove/e621-to-graph/pkg/util/queue" - "net/http" -) - -type schedulerTaskImpl[T e621.DataType] struct { - task e621.Task[T] - channel chan e621.DataResponse[T] - username string - apiKey string -} - -func (s schedulerTaskImpl[T]) BasicAuth() (string, string) { - return s.username, s.apiKey -} - -func NewSchedulerTaskImpl[T e621.DataType](task e621.Task[T], channel chan e621.DataResponse[T], username string, apiKey string) queue.SchedulerTask { - return &schedulerTaskImpl[T]{ - task: task, - channel: channel, - username: username, - apiKey: apiKey, - } -} - -func (s schedulerTaskImpl[T]) SendError(err error) { - s.channel <- s.task.HandleError(err) -} - -func (s schedulerTaskImpl[T]) UriPath() string { - return s.task.UriPath() -} - -func (s schedulerTaskImpl[T]) SendStatusCode(statusCode int) { - s.channel <- s.task.HandleStatusCode(statusCode) -} - -func (s schedulerTaskImpl[T]) SendResponse(response *http.Response) { - s.channel <- s.task.HandleResponse(response) -} diff --git a/internal/e621/task.go b/internal/e621/task.go deleted file mode 100644 index 16a9526..0000000 --- a/internal/e621/task.go +++ /dev/null @@ -1,70 +0,0 @@ -package e621 - -import ( - "encoding/json" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621" - pgkError "git.dragse.it/anthrove/e621-to-graph/pkg/error" - log "github.com/sirupsen/logrus" - "net/http" - "reflect" -) - -type e621APITask[T e621.DataType] struct { - uri string -} - -func NewE621ApiTask[T e621.DataType](uri string) e621.Task[T] { - log.WithFields(log.Fields{ - "uri": uri, - "type": reflect.TypeOf(*new(T)), - }).Debug("e621: task created") - return &e621APITask[T]{ - uri: uri, - } -} - -func (e e621APITask[T]) UriPath() string { - return e.uri -} - -func (e e621APITask[T]) HandleStatusCode(statusCode int) e621.DataResponse[T] { - var err error - switch statusCode { - case 421: - err = pgkError.RateLimitReachedError{} - case 424: - err = pgkError.InvalidParametersError{} - case 520: - err = pgkError.OriginConnectionTimeOutError{} - case 522: - err = pgkError.OriginConnectionTimeOutError{} - case 524: - err = pgkError.OriginConnectionTimeOutError{} - case 525: - err = pgkError.SSLHandshakeFailedError{} - default: - err = pgkError.StatusCodesToError(statusCode) - } - - log.WithFields(log.Fields{ - "status_code": statusCode, - }).Debug("e621: handel status code") - return e621.DataResponse[T]{Error: err} -} - -func (e e621APITask[T]) HandleResponse(responseData *http.Response) e621.DataResponse[T] { - var data T - log.WithFields(log.Fields{ - "type": reflect.TypeOf(*new(T)), - }).Debug("e621: decoding json response") - err := json.NewDecoder(responseData.Body).Decode(&data) - defer responseData.Body.Close() - if err != nil { - return e621.DataResponse[T]{Error: err} - } - return e621.DataResponse[T]{Data: data} -} - -func (e e621APITask[T]) HandleError(error error) e621.DataResponse[T] { - return e621.DataResponse[T]{Error: error} -} diff --git a/internal/e621/user.go b/internal/e621/user.go deleted file mode 100644 index 41ba62a..0000000 --- a/internal/e621/user.go +++ /dev/null @@ -1,19 +0,0 @@ -package e621 - -import ( - "fmt" - "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" - log "github.com/sirupsen/logrus" -) - -// GetUserInfo retrieves the users information from e621 API. -func (c *Client) GetUserInfo(username string) func() (model.E621User, error) { - URIPath := fmt.Sprintf("users/%s.json", username) - log.WithFields(log.Fields{ - "username": username, - "uri": URIPath, - }).Debug("e621: getting user info") - e621Task := NewE621ApiTask[model.E621User](URIPath) - return scheduler.Schedule[model.E621User](c.scheduler, e621Task, c.username, c.apiKey) -} diff --git a/internal/service/manager.go b/internal/service/manager.go index b7d093a..5e98d16 100644 --- a/internal/service/manager.go +++ b/internal/service/manager.go @@ -2,21 +2,18 @@ package service import ( "context" - "git.dragse.it/anthrove/e621-to-graph/internal/e621" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model" "git.dragse.it/anthrove/e621-to-graph/pkg/logic" "git.dragse.it/anthrove/e621-to-graph/pkg/util" log "github.com/sirupsen/logrus" "time" ) -func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621Client e621.Client, username string) error { +func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, client *e621.Client, username string) error { var err error - var page int64 = 1 - var allFavorites []model.Post - AwaitE621User := e621Client.GetUserInfo(username) - e621User, err := AwaitE621User() + e621User, err := client.GetUserByName(username).Execute() if err != nil { log.Info(err) return err @@ -47,29 +44,11 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621 }).Info("service: start processing favorites") start := time.Now() - for { - AwaitE621Favorites := e621Client.GetFavorites(ctx, e621User, page) - - favoritesPage, err := AwaitE621Favorites() - if err != nil { - log.Error(err.Error()) - } - - // Append the fetched posts to the result slice - allFavorites = append(allFavorites, favoritesPage.Posts...) - - // If no more posts are returned, return the accumulated favorites - if len(favoritesPage.Posts) == 0 { - break - } - - // Update the last post ID for the next page request - page += 1 - - } + e621FavoritesBuilder := client.GetFavoritesBuilder().SetUserID(e621User.ID) + e621Favorites, err := client.GetAllFavoritesForUser(e621FavoritesBuilder) // Uploads all Tags, Posts as Nodes to Neo4j - for i, post := range allFavorites { + for i, post := range e621Favorites { if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists { log.WithFields(log.Fields{ "e621_username": e621User.Name, @@ -90,7 +69,7 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621 "e621_username": e621User.Name, "e621_user_id": e621User.ID, "post_number": i, - "post_amount": len(allFavorites), + "post_amount": len(e621Favorites), "post_id": post.ID, "upload_time": time.Since(start), }).Debug("service: uploading post") @@ -135,7 +114,7 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621 "e621_username": e621User.Name, "e621_user_id": e621User.ID, "post_number": i, - "post_amount": len(allFavorites), + "post_amount": len(e621Favorites), "post_id": post.ID, "upload_time": time.Since(start), }).Debug("service: making relationship") @@ -143,7 +122,7 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621 log.WithFields(log.Fields{ "e621_username": e621User.Name, "e621_user_id": e621User.ID, - "post_amount": len(allFavorites), + "post_amount": len(e621Favorites), "scrape_time": time.Since(start), }).Info("service: finished processing favorites") @@ -217,7 +196,7 @@ func uploadNodes(ctx context.Context, graphConnection logic.GraphConnection, pos } // uploadPostToUserRelationship creates a relationship between the user and the post -func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.E621User) error { +func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.User) error { err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID) if err != nil { return err diff --git a/pkg/e621/model/post.go b/pkg/e621/model/post.go deleted file mode 100644 index ae307db..0000000 --- a/pkg/e621/model/post.go +++ /dev/null @@ -1,102 +0,0 @@ -package model - -import "encoding/json" - -func UnmarshalE621Post(data []byte) (PostResponseWrapper, error) { - var r PostResponseWrapper - err := json.Unmarshal(data, &r) - return r, err -} - -func (r *PostResponseWrapper) Marshal() ([]byte, error) { - return json.Marshal(r) -} - -type PostResponseWrapper struct { - Posts []Post `json:"posts"` -} - -type Post struct { - ID int64 `json:"id"` - CreatedAt string `json:"created_at"` - UpdatedAt string `json:"updated_at"` - File File `json:"file"` - Preview Preview `json:"preview"` - Sample Sample `json:"sample"` - Score Score `json:"score"` - Tags PostTags `json:"tags"` - LockedTags []interface{} `json:"locked_tags"` - ChangeSeq int64 `json:"change_seq"` - Flags Flags `json:"flags"` - Rating string `json:"rating"` - FavCount int64 `json:"fav_count"` - Sources []string `json:"sources"` - Pools []interface{} `json:"pools"` - Relationships Relationships `json:"relationships"` - ApproverID *int64 `json:"approver_id"` - UploaderID int64 `json:"uploader_id"` - Description string `json:"description"` - CommentCount int64 `json:"comment_count"` - IsFavorited bool `json:"is_favorited"` - HasNotes bool `json:"has_notes"` - Duration interface{} `json:"duration"` -} - -type File struct { - Width int64 `json:"width"` - Height int64 `json:"height"` - EXT string `json:"ext"` - Size int64 `json:"size"` - Md5 string `json:"md5"` - URL string `json:"url"` -} - -type Flags struct { - Pending bool `json:"pending"` - Flagged bool `json:"flagged"` - NoteLocked bool `json:"note_locked"` - StatusLocked bool `json:"status_locked"` - RatingLocked bool `json:"rating_locked"` - Deleted bool `json:"deleted"` -} - -type Preview struct { - Width int64 `json:"width"` - Height int64 `json:"height"` - URL string `json:"url"` -} - -type Relationships struct { - ParentID interface{} `json:"parent_id"` - HasChildren bool `json:"has_children"` - HasActiveChildren bool `json:"has_active_children"` - Children []interface{} `json:"children"` -} - -type Sample struct { - Has bool `json:"has"` - Height int64 `json:"height"` - Width int64 `json:"width"` - URL string `json:"url"` - Alternates Alternates `json:"alternates"` -} - -type Alternates struct { -} - -type Score struct { - Up int64 `json:"up"` - Down int64 `json:"down"` - Total int64 `json:"total"` -} - -type PostTags struct { - General []string `json:"general"` - Species []string `json:"species"` - Character []string `json:"character"` - Copyright []string `json:"copyright"` - Artist []string `json:"artist"` - Invalid []interface{} `json:"invalid"` - Lore []interface{} `json:"lore"` - Meta []string `json:"meta"` -} diff --git a/pkg/e621/model/user.go b/pkg/e621/model/user.go deleted file mode 100644 index 5113500..0000000 --- a/pkg/e621/model/user.go +++ /dev/null @@ -1,83 +0,0 @@ -package model - -import "encoding/json" - -func UnmarshalE621User(data []byte) (E621User, error) { - var r E621User - err := json.Unmarshal(data, &r) - return r, err -} - -func (r *E621User) Marshal() ([]byte, error) { - return json.Marshal(r) -} - -type E621User struct { - WikiPageVersionCount int64 `json:"wiki_page_version_count"` - ArtistVersionCount int64 `json:"artist_version_count"` - PoolVersionCount int64 `json:"pool_version_count"` - ForumPostCount int64 `json:"forum_post_count"` - CommentCount int64 `json:"comment_count"` - FlagCount int64 `json:"flag_count"` - FavoriteCount int64 `json:"favorite_count"` - PositiveFeedbackCount int64 `json:"positive_feedback_count"` - NeutralFeedbackCount int64 `json:"neutral_feedback_count"` - NegativeFeedbackCount int64 `json:"negative_feedback_count"` - UploadLimit int64 `json:"upload_limit"` - ID int64 `json:"id"` - CreatedAt string `json:"created_at"` - Name string `json:"name"` - Level int64 `json:"level"` - BaseUploadLimit int64 `json:"base_upload_limit"` - PostUploadCount int64 `json:"post_upload_count"` - PostUpdateCount int64 `json:"post_update_count"` - NoteUpdateCount int64 `json:"note_update_count"` - IsBanned bool `json:"is_banned"` - CanApprovePosts bool `json:"can_approve_posts"` - CanUploadFree bool `json:"can_upload_free"` - LevelString string `json:"level_string"` - AvatarID int64 `json:"avatar_id"` - ShowAvatars bool `json:"show_avatars"` - BlacklistAvatars bool `json:"blacklist_avatars"` - BlacklistUsers bool `json:"blacklist_users"` - DescriptionCollapsedInitially bool `json:"description_collapsed_initially"` - HideComments bool `json:"hide_comments"` - ShowHiddenComments bool `json:"show_hidden_comments"` - ShowPostStatistics bool `json:"show_post_statistics"` - HasMail bool `json:"has_mail"` - ReceiveEmailNotifications bool `json:"receive_email_notifications"` - EnableKeyboardNavigation bool `json:"enable_keyboard_navigation"` - EnablePrivacyMode bool `json:"enable_privacy_mode"` - StyleUsernames bool `json:"style_usernames"` - EnableAutoComplete bool `json:"enable_auto_complete"` - HasSavedSearches bool `json:"has_saved_searches"` - DisableCroppedThumbnails bool `json:"disable_cropped_thumbnails"` - DisableMobileGestures bool `json:"disable_mobile_gestures"` - EnableSafeMode bool `json:"enable_safe_mode"` - DisableResponsiveMode bool `json:"disable_responsive_mode"` - DisablePostTooltips bool `json:"disable_post_tooltips"` - NoFlagging bool `json:"no_flagging"` - NoFeedback bool `json:"no_feedback"` - DisableUserDmails bool `json:"disable_user_dmails"` - EnableCompactUploader bool `json:"enable_compact_uploader"` - ReplacementsBeta bool `json:"replacements_beta"` - IsBdStaff bool `json:"is_bd_staff"` - UpdatedAt string `json:"updated_at"` - Email string `json:"email"` - LastLoggedInAt string `json:"last_logged_in_at"` - LastForumReadAt string `json:"last_forum_read_at"` - RecentTags string `json:"recent_tags"` - CommentThreshold int64 `json:"comment_threshold"` - DefaultImageSize string `json:"default_image_size"` - FavoriteTags string `json:"favorite_tags"` - BlacklistedTags string `json:"blacklisted_tags"` - TimeZone string `json:"time_zone"` - PerPage int64 `json:"per_page"` - CustomStyle string `json:"custom_style"` - APIRegenMultiplier int64 `json:"api_regen_multiplier"` - APIBurstLimit int64 `json:"api_burst_limit"` - RemainingAPILimit int64 `json:"remaining_api_limit"` - StatementTimeout int64 `json:"statement_timeout"` - FavoriteLimit int64 `json:"favorite_limit"` - TagQueryLimit int64 `json:"tag_query_limit"` -} diff --git a/pkg/e621/task.go b/pkg/e621/task.go deleted file mode 100644 index 7e69ec5..0000000 --- a/pkg/e621/task.go +++ /dev/null @@ -1,22 +0,0 @@ -package e621 - -import ( - "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" - "net/http" -) - -type DataResponse[T DataType] struct { - Data T - Error error -} - -type DataType interface { - model.E621User | model.PostResponseWrapper -} - -type Task[T DataType] interface { - UriPath() string - HandleError(error error) DataResponse[T] - HandleStatusCode(statusCode int) DataResponse[T] - HandleResponse(responseData *http.Response) DataResponse[T] -} diff --git a/pkg/error/error.go b/pkg/error/error.go deleted file mode 100644 index 1e17d37..0000000 --- a/pkg/error/error.go +++ /dev/null @@ -1,105 +0,0 @@ -package error - -import "fmt" - -func StatusCodesToError(statusCode int) error { - var err error - switch statusCode { - case 403: - err = AccessDeniedError{} - case 404: - err = NotFoundError{} - case 412: - err = PreconditionFailedError{} - case 421: - err = RateLimitReachedError{} - case 424: - err = InvalidParametersError{} - case 500: - err = InternalServerError{} - case 502: - err = BadGatewayError{} - case 503: - err = ServiceUnavailableError{} - default: - err = fmt.Errorf("unhandels status code: %d", statusCode) - } - return err -} - -type AccessDeniedError struct { -} - -func (_ AccessDeniedError) Error() string { - return "access denied" -} - -type NotFoundError struct { -} - -func (_ NotFoundError) Error() string { - return "not found" -} - -type PreconditionFailedError struct { -} - -func (_ PreconditionFailedError) Error() string { - return "precondition failed" -} - -type RateLimitReachedError struct { -} - -func (_ RateLimitReachedError) Error() string { - return "rate limit reached" -} - -type InvalidParametersError struct { -} - -func (_ InvalidParametersError) Error() string { - return "invalide parameters" -} - -type InternalServerError struct { -} - -func (_ InternalServerError) Error() string { - return "internal server error" -} - -type BadGatewayError struct { -} - -func (_ BadGatewayError) Error() string { - return "bad gateway" -} - -type ServiceUnavailableError struct { -} - -func (_ ServiceUnavailableError) Error() string { - return "service unavailable" -} - -type UnknownError struct { -} - -func (_ UnknownError) Error() string { - return "unknown error" -} - -type OriginConnectionTimeOutError struct { -} - -func (_ OriginConnectionTimeOutError) Error() string { - return "origin connection time-out" -} - -type SSLHandshakeFailedError struct { -} - -func (_ SSLHandshakeFailedError) Error() string { - return "ssl handshake failed" -} diff --git a/pkg/logic/database.go b/pkg/logic/database.go index 4b3fc10..09e384c 100644 --- a/pkg/logic/database.go +++ b/pkg/logic/database.go @@ -2,17 +2,17 @@ package logic import ( "context" - "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" + "git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model" ) type GraphConnection interface { Connect(ctx context.Context, endpoint string, username string, password string) error - UploadUser(ctx context.Context, user model.E621User) error + UploadUser(ctx context.Context, user model.User) error UploadSource(ctx context.Context, SourceURL string) error - UploadPost(ctx context.Context, e621ID int64) error + UploadPost(ctx context.Context, e621ID model.PostID) error UploadTag(ctx context.Context, name string, tagType string) error - EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error - EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error - EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error - CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error) + EstablishPostToTagLink(ctx context.Context, e621PostID model.PostID, e621Tag string) error + EstablishPostToSourceLink(ctx context.Context, e621PostID model.PostID, sourceURL string) error + EstablishUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) error + CheckUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) (bool, error) }