implement_new_e621_sdk_#17 (#18)

As mentioned in #17, I implemented the new SDK. I removed the scheduler and executor code since they were no longer needed.

Reviewed-on: anthrove/e621-to-graph#18
Reviewed-by: Lennard Brinkhaus <lennard.brinkhaus@noreply.localhost>
Reviewed-by: daskadse <daskadse@noreply.localhost>
Co-authored-by: SoXX <soxx@fenpa.ws>
Co-committed-by: SoXX <soxx@fenpa.ws>

.gitea/workflows/build_check.yaml

@@ -14,7 +14,7 @@ jobs:
         uses: https://github.com/actions/setup-go@v3
         with:
           # The Go version to download (if necessary) and use. Supports semver spec and ranges.
-          go-version: 1.20 # optional
+          go-version: 1.21.3 # optional
           # Path to the go.mod file.
           go-version-file: ./go.mod # optional
           # Set this option to true if you want the action to always check for the latest available version that satisfies the version spec

build/package/Dockerfile

@@ -1,6 +1,6 @@
 FROM golang:alpine as builder
 
-WORKDIR /go/src/git.dragse.it/fenpaws/e621-to-neo4j
+WORKDIR /go/src/git.dragse.it/fenpaws/e621-to-graph
 
 RUN apk add -U --no-cache ca-certificates && update-ca-certificates
 
@@ -8,8 +8,8 @@ COPY go.mod go.sum ./
 RUN go mod download
 
 COPY . ./
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags "-w -s" -o /app
 
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build  -a -installsuffix cgo -ldflags "-w -s" -o /app ./cmd/scraper/
 
 FROM scratch
 

cmd/scraper/main.go

@@ -2,10 +2,10 @@ package main
 
 import (
 	"context"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621"
 	"git.dragse.it/anthrove/e621-to-graph/internal/api"
 	"git.dragse.it/anthrove/e621-to-graph/internal/config"
 	"git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j"
-	"git.dragse.it/anthrove/e621-to-graph/internal/e621"
 	"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
 	log "github.com/sirupsen/logrus"
 	"net/http"
@@ -42,9 +42,9 @@ func main() {
 	}).Info("main: database connection successful")
 
 	// Initialize the e621API
-	e621Client := e621.NewClient(appConfig.E621APIKey, appConfig.E621Username)
+	client := e621.NewClient(appConfig.E621Username, appConfig.E621APIKey)
 	// Register the ScapeUserFavourites with the "/user" route
-	http.HandleFunc("/user", api.ScapeUserFavourites(ctx, graphConnection, e621Client))
+	http.HandleFunc("/user", api.ScapeUserFavourites(ctx, graphConnection, &client))
 
 	// Start the HTTP server
 	err = http.ListenAndServe(":8080", nil)

deployments/docker-compose.yaml

@@ -1,7 +1,10 @@
 services:
   app:
     restart: unless-stopped
-    build: .
+    image: anthrove/e621-to-graph:latest
+    build:
+      context: ../.
+      dockerfile: build/package/Dockerfile
     ports:
       - 8080:8080
     env_file: .env

go.mod

@@ -1,12 +1,17 @@
 module git.dragse.it/anthrove/e621-to-graph
 
-go 1.20
+go 1.21.3
 
 require (
+	git.dragse.it/anthrove/e621-sdk-go v1.3.0
 	github.com/caarlos0/env v3.5.0+incompatible
 	github.com/neo4j/neo4j-go-driver/v5 v5.8.1
 	github.com/sirupsen/logrus v1.9.3
-	golang.org/x/time v0.3.0
+
 )
 
-require golang.org/x/sys v0.9.0 // indirect
+require (
+	github.com/joho/godotenv v1.5.1 // indirect
+	golang.org/x/sys v0.9.0 // indirect
+	golang.org/x/time v0.3.0 // indirect
+)

go.sum

@@ -1,8 +1,14 @@
+git.dragse.it/anthrove/e621-sdk-go v1.3.0 h1:sbjrOps4WxXf42kyNb8ZVxC6dCiFrCkqv4C8BgKkIGA=
+git.dragse.it/anthrove/e621-sdk-go v1.3.0/go.mod h1:urFl0jjx2UK8Vz1tMI253CvWK8fZqd/a9+xdE8ZBwq4=
 github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs=
 github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/jarcoal/httpmock v1.3.1 h1:iUx3whfZWVf3jT01hQTO/Eo5sAYtB2/rqaUuOtpInww=
+github.com/jarcoal/httpmock v1.3.1/go.mod h1:3yb8rc4BI7TCBhFY8ng0gjuLKJNquuDNiPaZjnENuYg=
+github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
+github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/neo4j/neo4j-go-driver/v5 v5.8.1 h1:IysKg6KJIUgyItmnHRRrt2N8srbd6znMslRW3qQErTQ=
 github.com/neo4j/neo4j-go-driver/v5 v5.8.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

internal/api/user.go

@@ -3,7 +3,7 @@ package api
 import (
 	"context"
 	"fmt"
-	"git.dragse.it/anthrove/e621-to-graph/internal/e621"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621"
 	"git.dragse.it/anthrove/e621-to-graph/internal/service"
 	"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
 	log "github.com/sirupsen/logrus"
@@ -11,7 +11,7 @@ import (
 )
 
 // ScapeUserFavourites is the handler for the user API
-func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
+func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnection, client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method != http.MethodPost {
 			w.WriteHeader(http.StatusMethodNotAllowed)
@@ -27,7 +27,7 @@ func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnect
 		}
 
 		// Perform further processing with the username
-		go service.ScrapeUser(ctx, graphConnection, *e621Client, username)
+		go service.ScrapeUser(ctx, graphConnection, client, username)
 
 		// Send a response
 		w.WriteHeader(http.StatusOK)

internal/database/neo4j/impl.go

@@ -2,7 +2,7 @@ package neo4j
 
 import (
 	"context"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model"
 	"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j/config"
@@ -20,19 +20,19 @@ func NewNeo4JConnection(neo4jDebug bool) logic.GraphConnection {
 	}
 }
 
-func (c *neo4jConnection) CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error) {
+func (c *neo4jConnection) CheckUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) (bool, error) {
-	return RelationshipCheckUserToPost(ctx, c.driver, e621PostID, e621UserID)
+	return CheckUserToPostLink(ctx, c.driver, e621PostID, e621UserID)
 }
 
-func (c *neo4jConnection) EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error {
+func (c *neo4jConnection) EstablishPostToTagLink(ctx context.Context, e621PostID model.PostID, e621Tag string) error {
 	return EstablishPostTagLink(ctx, c.driver, e621PostID, e621Tag)
 }
 
-func (c *neo4jConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error {
+func (c *neo4jConnection) EstablishPostToSourceLink(ctx context.Context, e621PostID model.PostID, sourceURL string) error {
 	return EstablishPostToSourceLink(ctx, c.driver, e621PostID, sourceURL)
 }
 
-func (c *neo4jConnection) EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error {
+func (c *neo4jConnection) EstablishUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) error {
 	return EstablishUserToPostLink(ctx, c.driver, e621PostID, e621UserID)
 }
 
@@ -40,15 +40,15 @@ func (c *neo4jConnection) UploadTag(ctx context.Context, name string, tagType st
 	return CreateTagNode(ctx, c.driver, name, tagType)
 }
 
-func (c *neo4jConnection) UploadPost(ctx context.Context, e621ID int64) error {
+func (c *neo4jConnection) UploadPost(ctx context.Context, postID model.PostID) error {
-	return CreatePostNode(ctx, c.driver, e621ID)
+	return CreatePostNode(ctx, c.driver, postID)
 }
 
 func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) error {
 	return CreateSourceNode(ctx, c.driver, SourceURL)
 }
 
-func (c *neo4jConnection) UploadUser(ctx context.Context, user model.E621User) error {
+func (c *neo4jConnection) UploadUser(ctx context.Context, user model.User) error {
 	return CreateUserNode(ctx, c.driver, user)
 }
 func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error {

internal/database/neo4j/post.go

@@ -2,16 +2,17 @@ package neo4j
 
 import (
 	"context"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 )
 
-func CreatePostNode(ctx context.Context, driver neo4j.DriverWithContext, e621ID int64) error {
+func CreatePostNode(ctx context.Context, driver neo4j.DriverWithContext, postID model.PostID) error {
 	query := `
-		MERGE (u:e621Post {e621PostID: $e621ID})
+		MERGE (u:e621Post {e621PostID: $postID})
 		RETURN u
 `
 	params := map[string]any{
-		"e621ID": e621ID,
+		"postID": postID,
 	}
 
 	_, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)

internal/database/neo4j/relationship.go

@@ -2,11 +2,12 @@ package neo4j
 
 import (
 	"context"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 	log "github.com/sirupsen/logrus"
 )
 
-func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621Tag string) error {
+func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, e621Tag string) error {
 	query := `
 		MATCH (p:e621Post {e621PostID: $e621PostID})
 		MATCH (t:e621Tag {e621Tag: $e621Tag})
@@ -29,7 +30,7 @@ func EstablishPostTagLink(ctx context.Context, driver neo4j.DriverWithContext, e
 	return nil
 }
 
-func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, sourceURL string) error {
+func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, sourceURL string) error {
 	query := `
 		MATCH (p:e621Post {e621PostID: $e621PostID})
 		MATCH (s:Source {URL: $sourceURL})
@@ -52,7 +53,7 @@ func EstablishPostToSourceLink(ctx context.Context, driver neo4j.DriverWithConte
 	return nil
 }
 
-func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) error {
+func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, e621UserID model.UserID) error {
 	query := `
 		MATCH (p:e621Post {e621PostID: $e621PostID})
 		MATCH (u:e621User {e621ID: $e621ID})
@@ -74,8 +75,8 @@ func EstablishUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext
 	return nil
 }
 
-// CheckUserToPostRelationship gives back a bool if the connection between the post and the user exists
+// CheckUserToPostLink gives back a bool if the connection between the post and the user exists
-func RelationshipCheckUserToPost(ctx context.Context, driver neo4j.DriverWithContext, e621PostID int64, e621UserID int64) (bool, error) {
+func CheckUserToPostLink(ctx context.Context, driver neo4j.DriverWithContext, e621PostID model.PostID, e621UserID model.UserID) (bool, error) {
 	query := `
 		MATCH (user:e621User {e621ID: $e621ID})-[favorite:IS_FAVORITE]->(post:e621Post {e621PostID: $e621PostID})
 		RETURN COUNT(favorite) > 0 AS isFavorite

internal/database/neo4j/user.go

@@ -2,11 +2,11 @@ package neo4j
 
 import (
 	"context"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model"
 	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
 )
 
-func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user model.E621User) error {
+func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user model.User) error {
 	query := `
 		MERGE (u:e621User {e621ID: $id, e621Username: $name})
 		RETURN u

internal/e621/client.go

@@ -1,24 +0,0 @@
-package e621
-
-import (
-	e621 "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
-	"golang.org/x/time/rate"
-)
-
-// Client represents the e621 API client.
-type Client struct {
-	apiKey    string
-	username  string
-	scheduler *e621.Scheduler
-}
-
-// NewClient creates a new e621 API client.
-func NewClient(apiKey string, username string) *Client {
-	scheduler := e621.NewScheduler()
-	scheduler.SetLimiter(rate.NewLimiter(1, 2))
-	return &Client{
-		apiKey:    apiKey,
-		username:  username,
-		scheduler: scheduler,
-	}
-}

internal/e621/favorites.go

@@ -1,22 +0,0 @@
-package e621
-
-import (
-	"context"
-	"fmt"
-	"git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
-	log "github.com/sirupsen/logrus"
-)
-
-// GetFavorites retrieves all favorites from the e621 API.
-func (c *Client) GetFavorites(_ context.Context, user model.E621User, page int64) func() (model.PostResponseWrapper, error) {
-	URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page)
-	log.WithFields(log.Fields{
-		"user": user.Name,
-		"id":   user.ID,
-		"page": page,
-		"uri":  URIPath,
-	}).Trace("e621: adding task to get favorites")
-	e621Task := NewE621ApiTask[model.PostResponseWrapper](URIPath)
-	return scheduler.Schedule[model.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey)
-}

internal/e621/scheduler/executor.go

@@ -1,43 +0,0 @@
-package scheduler
-
-import (
-	"fmt"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
-	log "github.com/sirupsen/logrus"
-	"net/http"
-)
-
-const (
-	BASEURL = "https://e621.net"
-)
-
-func GetAPIRequest(schedulerTask queue.SchedulerTask) {
-	var err error
-	log.Debug("executing scheduler task")
-
-	url := fmt.Sprintf("%s/%s", BASEURL, schedulerTask.UriPath())
-	req, err := http.NewRequest("GET", url, nil)
-	if err != nil {
-		schedulerTask.SendError(err)
-		return
-	}
-
-	req.Header.Set("User-Agent", "FavGetter (by Selloo)")
-	req.Header.Add("Accept", "application/json")
-	req.SetBasicAuth(schedulerTask.BasicAuth())
-
-	client := http.Client{}
-
-	resp, err := client.Do(req)
-	if err != nil {
-		schedulerTask.SendStatusCode(resp.StatusCode)
-		return
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		schedulerTask.SendStatusCode(resp.StatusCode)
-		return
-	}
-
-	schedulerTask.SendResponse(resp)
-}

internal/e621/scheduler/scheduler.go

@@ -1,75 +0,0 @@
-package scheduler
-
-import (
-	"context"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/time/rate"
-	"reflect"
-)
-
-type Scheduler struct {
-	queue   queue.Queue
-	limiter *rate.Limiter
-}
-
-func NewScheduler() *Scheduler {
-	scheduler := &Scheduler{
-		queue:   queue.NewQueue(),
-		limiter: nil,
-	}
-	go scheduler.StartExecutionHandler()
-	return scheduler
-}
-
-func Schedule[T e621.DataType](s *Scheduler, t e621.Task[T], username string, apiKey string) func() (T, error) {
-	channel := make(chan e621.DataResponse[T])
-	schedulerTask := NewSchedulerTaskImpl[T](t, channel, username, apiKey)
-	log.WithFields(log.Fields{
-		"type": reflect.TypeOf(*new(T)),
-	}).Debug("scheduler: pushing task to queue")
-
-	err := s.queue.Push(schedulerTask)
-	if err != nil {
-		return func() (T, error) {
-			var nil T
-			return nil, err
-		}
-	}
-
-	log.WithFields(log.Fields{
-		"type": reflect.TypeOf(*new(T)),
-	}).Trace("scheduler: pushed task to queue")
-
-	return func() (T, error) {
-		log.Trace("scheduler: getting data from channel")
-		data := <-channel
-		return data.Data, data.Error
-	}
-}
-
-func (s *Scheduler) SetLimiter(limiter *rate.Limiter) {
-	log.WithFields(log.Fields{
-		"burst":  limiter.Burst(),
-		"limit":  limiter.Limit(),
-		"tokens": limiter.Tokens(),
-	}).Debug("scheduler: setting limiter")
-	s.limiter = limiter
-}
-
-func (s *Scheduler) StartExecutionHandler() {
-	for {
-		if s.limiter != nil {
-			s.limiter.Wait(context.Background())
-		}
-		s.queue.WaitForElement()
-		log.Trace("scheduler: element found")
-		task, err := s.queue.Pop()
-		if err != nil {
-			continue
-		}
-		GetAPIRequest(task)
-	}
-
-}

internal/e621/scheduler/task.go

@@ -1,43 +0,0 @@
-package scheduler
-
-import (
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
-	"net/http"
-)
-
-type schedulerTaskImpl[T e621.DataType] struct {
-	task     e621.Task[T]
-	channel  chan e621.DataResponse[T]
-	username string
-	apiKey   string
-}
-
-func (s schedulerTaskImpl[T]) BasicAuth() (string, string) {
-	return s.username, s.apiKey
-}
-
-func NewSchedulerTaskImpl[T e621.DataType](task e621.Task[T], channel chan e621.DataResponse[T], username string, apiKey string) queue.SchedulerTask {
-	return &schedulerTaskImpl[T]{
-		task:     task,
-		channel:  channel,
-		username: username,
-		apiKey:   apiKey,
-	}
-}
-
-func (s schedulerTaskImpl[T]) SendError(err error) {
-	s.channel <- s.task.HandleError(err)
-}
-
-func (s schedulerTaskImpl[T]) UriPath() string {
-	return s.task.UriPath()
-}
-
-func (s schedulerTaskImpl[T]) SendStatusCode(statusCode int) {
-	s.channel <- s.task.HandleStatusCode(statusCode)
-}
-
-func (s schedulerTaskImpl[T]) SendResponse(response *http.Response) {
-	s.channel <- s.task.HandleResponse(response)
-}

internal/e621/task.go

@@ -1,70 +0,0 @@
-package e621
-
-import (
-	"encoding/json"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621"
-	pgkError "git.dragse.it/anthrove/e621-to-graph/pkg/error"
-	log "github.com/sirupsen/logrus"
-	"net/http"
-	"reflect"
-)
-
-type e621APITask[T e621.DataType] struct {
-	uri string
-}
-
-func NewE621ApiTask[T e621.DataType](uri string) e621.Task[T] {
-	log.WithFields(log.Fields{
-		"uri":  uri,
-		"type": reflect.TypeOf(*new(T)),
-	}).Debug("e621: task created")
-	return &e621APITask[T]{
-		uri: uri,
-	}
-}
-
-func (e e621APITask[T]) UriPath() string {
-	return e.uri
-}
-
-func (e e621APITask[T]) HandleStatusCode(statusCode int) e621.DataResponse[T] {
-	var err error
-	switch statusCode {
-	case 421:
-		err = pgkError.RateLimitReachedError{}
-	case 424:
-		err = pgkError.InvalidParametersError{}
-	case 520:
-		err = pgkError.OriginConnectionTimeOutError{}
-	case 522:
-		err = pgkError.OriginConnectionTimeOutError{}
-	case 524:
-		err = pgkError.OriginConnectionTimeOutError{}
-	case 525:
-		err = pgkError.SSLHandshakeFailedError{}
-	default:
-		err = pgkError.StatusCodesToError(statusCode)
-	}
-
-	log.WithFields(log.Fields{
-		"status_code": statusCode,
-	}).Debug("e621: handel status code")
-	return e621.DataResponse[T]{Error: err}
-}
-
-func (e e621APITask[T]) HandleResponse(responseData *http.Response) e621.DataResponse[T] {
-	var data T
-	log.WithFields(log.Fields{
-		"type": reflect.TypeOf(*new(T)),
-	}).Debug("e621: decoding json response")
-	err := json.NewDecoder(responseData.Body).Decode(&data)
-	defer responseData.Body.Close()
-	if err != nil {
-		return e621.DataResponse[T]{Error: err}
-	}
-	return e621.DataResponse[T]{Data: data}
-}
-
-func (e e621APITask[T]) HandleError(error error) e621.DataResponse[T] {
-	return e621.DataResponse[T]{Error: error}
-}

internal/e621/user.go

@@ -1,19 +0,0 @@
-package e621
-
-import (
-	"fmt"
-	"git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
-	log "github.com/sirupsen/logrus"
-)
-
-// GetUserInfo retrieves the users information from e621 API.
-func (c *Client) GetUserInfo(username string) func() (model.E621User, error) {
-	URIPath := fmt.Sprintf("users/%s.json", username)
-	log.WithFields(log.Fields{
-		"username": username,
-		"uri":      URIPath,
-	}).Debug("e621: getting user info")
-	e621Task := NewE621ApiTask[model.E621User](URIPath)
-	return scheduler.Schedule[model.E621User](c.scheduler, e621Task, c.username, c.apiKey)
-}

internal/service/manager.go

@@ -2,21 +2,18 @@ package service
 
 import (
 	"context"
-	"git.dragse.it/anthrove/e621-to-graph/internal/e621"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model"
 	"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
 	"git.dragse.it/anthrove/e621-to-graph/pkg/util"
 	log "github.com/sirupsen/logrus"
 	"time"
 )
 
-func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621Client e621.Client, username string) error {
+func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, client *e621.Client, username string) error {
 	var err error
-	var page int64 = 1
-	var allFavorites []model.Post
 
-	AwaitE621User := e621Client.GetUserInfo(username)
+	e621User, err := client.GetUserByName(username).Execute()
-	e621User, err := AwaitE621User()
 	if err != nil {
 		log.Info(err)
 		return err
@@ -47,29 +44,11 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621
 	}).Info("service: start processing favorites")
 	start := time.Now()
 
-	for {
+	e621FavoritesBuilder := client.GetFavoritesBuilder().SetUserID(e621User.ID)
-		AwaitE621Favorites := e621Client.GetFavorites(ctx, e621User, page)
+	e621Favorites, err := client.GetAllFavoritesForUser(e621FavoritesBuilder)
-
-		favoritesPage, err := AwaitE621Favorites()
-		if err != nil {
-			log.Error(err.Error())
-		}
-
-		// Append the fetched posts to the result slice
-		allFavorites = append(allFavorites, favoritesPage.Posts...)
-
-		// If no more posts are returned, return the accumulated favorites
-		if len(favoritesPage.Posts) == 0 {
-			break
-		}
-
-		// Update the last post ID for the next page request
-		page += 1
-
-	}
 
 	// Uploads all Tags, Posts as Nodes to Neo4j
-	for i, post := range allFavorites {
+	for i, post := range e621Favorites {
 		if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists {
 			log.WithFields(log.Fields{
 				"e621_username": e621User.Name,
@@ -90,7 +69,7 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621
 			"e621_username": e621User.Name,
 			"e621_user_id":  e621User.ID,
 			"post_number":   i,
-			"post_amount":   len(allFavorites),
+			"post_amount":   len(e621Favorites),
 			"post_id":       post.ID,
 			"upload_time":   time.Since(start),
 		}).Debug("service: uploading post")
@@ -135,7 +114,7 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621
 			"e621_username": e621User.Name,
 			"e621_user_id":  e621User.ID,
 			"post_number":   i,
-			"post_amount":   len(allFavorites),
+			"post_amount":   len(e621Favorites),
 			"post_id":       post.ID,
 			"upload_time":   time.Since(start),
 		}).Debug("service: making relationship")
@@ -143,7 +122,7 @@ func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621
 	log.WithFields(log.Fields{
 		"e621_username": e621User.Name,
 		"e621_user_id":  e621User.ID,
-		"post_amount":   len(allFavorites),
+		"post_amount":   len(e621Favorites),
 		"scrape_time":   time.Since(start),
 	}).Info("service: finished processing favorites")
 
@@ -217,7 +196,7 @@ func uploadNodes(ctx context.Context, graphConnection logic.GraphConnection, pos
 }
 
 // uploadPostToUserRelationship creates a relationship between the user and the post
-func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.E621User) error {
+func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.User) error {
 	err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID)
 	if err != nil {
 		return err

pkg/e621/model/post.go

@@ -1,102 +0,0 @@
-package model
-
-import "encoding/json"
-
-func UnmarshalE621Post(data []byte) (PostResponseWrapper, error) {
-	var r PostResponseWrapper
-	err := json.Unmarshal(data, &r)
-	return r, err
-}
-
-func (r *PostResponseWrapper) Marshal() ([]byte, error) {
-	return json.Marshal(r)
-}
-
-type PostResponseWrapper struct {
-	Posts []Post `json:"posts"`
-}
-
-type Post struct {
-	ID            int64         `json:"id"`
-	CreatedAt     string        `json:"created_at"`
-	UpdatedAt     string        `json:"updated_at"`
-	File          File          `json:"file"`
-	Preview       Preview       `json:"preview"`
-	Sample        Sample        `json:"sample"`
-	Score         Score         `json:"score"`
-	Tags          PostTags      `json:"tags"`
-	LockedTags    []interface{} `json:"locked_tags"`
-	ChangeSeq     int64         `json:"change_seq"`
-	Flags         Flags         `json:"flags"`
-	Rating        string        `json:"rating"`
-	FavCount      int64         `json:"fav_count"`
-	Sources       []string      `json:"sources"`
-	Pools         []interface{} `json:"pools"`
-	Relationships Relationships `json:"relationships"`
-	ApproverID    *int64        `json:"approver_id"`
-	UploaderID    int64         `json:"uploader_id"`
-	Description   string        `json:"description"`
-	CommentCount  int64         `json:"comment_count"`
-	IsFavorited   bool          `json:"is_favorited"`
-	HasNotes      bool          `json:"has_notes"`
-	Duration      interface{}   `json:"duration"`
-}
-
-type File struct {
-	Width  int64  `json:"width"`
-	Height int64  `json:"height"`
-	EXT    string `json:"ext"`
-	Size   int64  `json:"size"`
-	Md5    string `json:"md5"`
-	URL    string `json:"url"`
-}
-
-type Flags struct {
-	Pending      bool `json:"pending"`
-	Flagged      bool `json:"flagged"`
-	NoteLocked   bool `json:"note_locked"`
-	StatusLocked bool `json:"status_locked"`
-	RatingLocked bool `json:"rating_locked"`
-	Deleted      bool `json:"deleted"`
-}
-
-type Preview struct {
-	Width  int64  `json:"width"`
-	Height int64  `json:"height"`
-	URL    string `json:"url"`
-}
-
-type Relationships struct {
-	ParentID          interface{}   `json:"parent_id"`
-	HasChildren       bool          `json:"has_children"`
-	HasActiveChildren bool          `json:"has_active_children"`
-	Children          []interface{} `json:"children"`
-}
-
-type Sample struct {
-	Has        bool       `json:"has"`
-	Height     int64      `json:"height"`
-	Width      int64      `json:"width"`
-	URL        string     `json:"url"`
-	Alternates Alternates `json:"alternates"`
-}
-
-type Alternates struct {
-}
-
-type Score struct {
-	Up    int64 `json:"up"`
-	Down  int64 `json:"down"`
-	Total int64 `json:"total"`
-}
-
-type PostTags struct {
-	General   []string      `json:"general"`
-	Species   []string      `json:"species"`
-	Character []string      `json:"character"`
-	Copyright []string      `json:"copyright"`
-	Artist    []string      `json:"artist"`
-	Invalid   []interface{} `json:"invalid"`
-	Lore      []interface{} `json:"lore"`
-	Meta      []string      `json:"meta"`
-}

pkg/e621/model/user.go

@@ -1,83 +0,0 @@
-package model
-
-import "encoding/json"
-
-func UnmarshalE621User(data []byte) (E621User, error) {
-	var r E621User
-	err := json.Unmarshal(data, &r)
-	return r, err
-}
-
-func (r *E621User) Marshal() ([]byte, error) {
-	return json.Marshal(r)
-}
-
-type E621User struct {
-	WikiPageVersionCount          int64  `json:"wiki_page_version_count"`
-	ArtistVersionCount            int64  `json:"artist_version_count"`
-	PoolVersionCount              int64  `json:"pool_version_count"`
-	ForumPostCount                int64  `json:"forum_post_count"`
-	CommentCount                  int64  `json:"comment_count"`
-	FlagCount                     int64  `json:"flag_count"`
-	FavoriteCount                 int64  `json:"favorite_count"`
-	PositiveFeedbackCount         int64  `json:"positive_feedback_count"`
-	NeutralFeedbackCount          int64  `json:"neutral_feedback_count"`
-	NegativeFeedbackCount         int64  `json:"negative_feedback_count"`
-	UploadLimit                   int64  `json:"upload_limit"`
-	ID                            int64  `json:"id"`
-	CreatedAt                     string `json:"created_at"`
-	Name                          string `json:"name"`
-	Level                         int64  `json:"level"`
-	BaseUploadLimit               int64  `json:"base_upload_limit"`
-	PostUploadCount               int64  `json:"post_upload_count"`
-	PostUpdateCount               int64  `json:"post_update_count"`
-	NoteUpdateCount               int64  `json:"note_update_count"`
-	IsBanned                      bool   `json:"is_banned"`
-	CanApprovePosts               bool   `json:"can_approve_posts"`
-	CanUploadFree                 bool   `json:"can_upload_free"`
-	LevelString                   string `json:"level_string"`
-	AvatarID                      int64  `json:"avatar_id"`
-	ShowAvatars                   bool   `json:"show_avatars"`
-	BlacklistAvatars              bool   `json:"blacklist_avatars"`
-	BlacklistUsers                bool   `json:"blacklist_users"`
-	DescriptionCollapsedInitially bool   `json:"description_collapsed_initially"`
-	HideComments                  bool   `json:"hide_comments"`
-	ShowHiddenComments            bool   `json:"show_hidden_comments"`
-	ShowPostStatistics            bool   `json:"show_post_statistics"`
-	HasMail                       bool   `json:"has_mail"`
-	ReceiveEmailNotifications     bool   `json:"receive_email_notifications"`
-	EnableKeyboardNavigation      bool   `json:"enable_keyboard_navigation"`
-	EnablePrivacyMode             bool   `json:"enable_privacy_mode"`
-	StyleUsernames                bool   `json:"style_usernames"`
-	EnableAutoComplete            bool   `json:"enable_auto_complete"`
-	HasSavedSearches              bool   `json:"has_saved_searches"`
-	DisableCroppedThumbnails      bool   `json:"disable_cropped_thumbnails"`
-	DisableMobileGestures         bool   `json:"disable_mobile_gestures"`
-	EnableSafeMode                bool   `json:"enable_safe_mode"`
-	DisableResponsiveMode         bool   `json:"disable_responsive_mode"`
-	DisablePostTooltips           bool   `json:"disable_post_tooltips"`
-	NoFlagging                    bool   `json:"no_flagging"`
-	NoFeedback                    bool   `json:"no_feedback"`
-	DisableUserDmails             bool   `json:"disable_user_dmails"`
-	EnableCompactUploader         bool   `json:"enable_compact_uploader"`
-	ReplacementsBeta              bool   `json:"replacements_beta"`
-	IsBdStaff                     bool   `json:"is_bd_staff"`
-	UpdatedAt                     string `json:"updated_at"`
-	Email                         string `json:"email"`
-	LastLoggedInAt                string `json:"last_logged_in_at"`
-	LastForumReadAt               string `json:"last_forum_read_at"`
-	RecentTags                    string `json:"recent_tags"`
-	CommentThreshold              int64  `json:"comment_threshold"`
-	DefaultImageSize              string `json:"default_image_size"`
-	FavoriteTags                  string `json:"favorite_tags"`
-	BlacklistedTags               string `json:"blacklisted_tags"`
-	TimeZone                      string `json:"time_zone"`
-	PerPage                       int64  `json:"per_page"`
-	CustomStyle                   string `json:"custom_style"`
-	APIRegenMultiplier            int64  `json:"api_regen_multiplier"`
-	APIBurstLimit                 int64  `json:"api_burst_limit"`
-	RemainingAPILimit             int64  `json:"remaining_api_limit"`
-	StatementTimeout              int64  `json:"statement_timeout"`
-	FavoriteLimit                 int64  `json:"favorite_limit"`
-	TagQueryLimit                 int64  `json:"tag_query_limit"`
-}

pkg/e621/task.go

@@ -1,22 +0,0 @@
-package e621
-
-import (
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
-	"net/http"
-)
-
-type DataResponse[T DataType] struct {
-	Data  T
-	Error error
-}
-
-type DataType interface {
-	model.E621User | model.PostResponseWrapper
-}
-
-type Task[T DataType] interface {
-	UriPath() string
-	HandleError(error error) DataResponse[T]
-	HandleStatusCode(statusCode int) DataResponse[T]
-	HandleResponse(responseData *http.Response) DataResponse[T]
-}

pkg/error/error.go

@@ -1,105 +0,0 @@
-package error
-
-import "fmt"
-
-func StatusCodesToError(statusCode int) error {
-	var err error
-	switch statusCode {
-	case 403:
-		err = AccessDeniedError{}
-	case 404:
-		err = NotFoundError{}
-	case 412:
-		err = PreconditionFailedError{}
-	case 421:
-		err = RateLimitReachedError{}
-	case 424:
-		err = InvalidParametersError{}
-	case 500:
-		err = InternalServerError{}
-	case 502:
-		err = BadGatewayError{}
-	case 503:
-		err = ServiceUnavailableError{}
-	default:
-		err = fmt.Errorf("unhandels status code: %d", statusCode)
-	}
-	return err
-}
-
-type AccessDeniedError struct {
-}
-
-func (_ AccessDeniedError) Error() string {
-	return "access denied"
-}
-
-type NotFoundError struct {
-}
-
-func (_ NotFoundError) Error() string {
-	return "not found"
-}
-
-type PreconditionFailedError struct {
-}
-
-func (_ PreconditionFailedError) Error() string {
-	return "precondition failed"
-}
-
-type RateLimitReachedError struct {
-}
-
-func (_ RateLimitReachedError) Error() string {
-	return "rate limit reached"
-}
-
-type InvalidParametersError struct {
-}
-
-func (_ InvalidParametersError) Error() string {
-	return "invalide parameters"
-}
-
-type InternalServerError struct {
-}
-
-func (_ InternalServerError) Error() string {
-	return "internal server error"
-}
-
-type BadGatewayError struct {
-}
-
-func (_ BadGatewayError) Error() string {
-	return "bad gateway"
-}
-
-type ServiceUnavailableError struct {
-}
-
-func (_ ServiceUnavailableError) Error() string {
-	return "service unavailable"
-}
-
-type UnknownError struct {
-}
-
-func (_ UnknownError) Error() string {
-	return "unknown error"
-}
-
-type OriginConnectionTimeOutError struct {
-}
-
-func (_ OriginConnectionTimeOutError) Error() string {
-	return "origin connection time-out"
-}
-
-type SSLHandshakeFailedError struct {
-}
-
-func (_ SSLHandshakeFailedError) Error() string {
-	return "ssl handshake failed"
-}

pkg/logic/database.go

@@ -2,17 +2,17 @@ package logic
 
 import (
 	"context"
-	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model"
 )
 
 type GraphConnection interface {
 	Connect(ctx context.Context, endpoint string, username string, password string) error
-	UploadUser(ctx context.Context, user model.E621User) error
+	UploadUser(ctx context.Context, user model.User) error
 	UploadSource(ctx context.Context, SourceURL string) error
-	UploadPost(ctx context.Context, e621ID int64) error
+	UploadPost(ctx context.Context, e621ID model.PostID) error
 	UploadTag(ctx context.Context, name string, tagType string) error
-	EstablishPostToTagLink(ctx context.Context, e621PostID int64, e621Tag string) error
+	EstablishPostToTagLink(ctx context.Context, e621PostID model.PostID, e621Tag string) error
-	EstablishPostToSourceLink(ctx context.Context, e621PostID int64, sourceURL string) error
+	EstablishPostToSourceLink(ctx context.Context, e621PostID model.PostID, sourceURL string) error
-	EstablishUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) error
+	EstablishUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) error
-	CheckUserToPostLink(ctx context.Context, e621PostID int64, e621UserID int64) (bool, error)
+	CheckUserToPostLink(ctx context.Context, e621PostID model.PostID, e621UserID model.UserID) (bool, error)
 }