From 802764092e2dd944fbff3bd31dec9e374cb011d9 Mon Sep 17 00:00:00 2001 From: SoXX Date: Mon, 17 Jul 2023 10:57:23 +0000 Subject: [PATCH] refactor_folder_structure_#11 (#14) As mentioned in Issue #11, the folder structure got an overall as some file names Co-authored-by: Fenpaws Reviewed-on: https://git.dragse.it/anthrove/e621-to-graph/pulls/14 Reviewed-by: Lennard Brinkhaus Co-authored-by: SoXX Co-committed-by: SoXX --- .gitea/workflows/build_check.yaml | 6 +- README.MD | 7 ++- Dockerfile => build/package/Dockerfile | 0 build.bat => build/package/build.bat | 0 main.go => cmd/scraper/main.go | 18 +++--- .../docker-compose.old.yml | 0 .../docker-compose.yaml | 0 e621/favorite.go | 14 ----- e621/task.go | 56 ------------------ e621/users.go | 13 ----- go.mod | 2 +- {api => internal/api}/user.go | 12 ++-- {utils => internal/config}/config.go | 2 +- {database => internal/database}/neo4j/impl.go | 8 +-- .../database/neo4j/model}/tag.go | 2 +- {database => internal/database}/neo4j/post.go | 0 .../database}/neo4j/relationship.go | 0 .../database}/neo4j/source.go | 0 {database => internal/database}/neo4j/tag.go | 8 +-- {database => internal/database}/neo4j/user.go | 4 +- {e621 => internal/e621}/client.go | 11 ++-- internal/e621/favorites.go | 15 +++++ .../e621/scheduler/executor.go | 12 ++-- .../e621/scheduler}/scheduler.go | 15 ++--- .../e621/scheduler/task.go | 13 +++-- internal/e621/task.go | 57 +++++++++++++++++++ internal/e621/user.go | 14 +++++ {services => internal/service}/manager.go | 36 ++++++------ {e621/models => pkg/e621/model}/post.go | 2 +- {e621/models => pkg/e621/model}/user.go | 2 +- utils/scheduler.go => pkg/e621/task.go | 14 +---- {utils => pkg/error}/error.go | 2 +- database/logic.go => pkg/logic/database.go | 15 +---- {utils => pkg/util}/list.go | 2 +- {utils => pkg/util/queue}/queue.go | 2 +- {utils => pkg/util/queue}/queue_test.go | 2 +- pkg/util/queue/scheduler.go | 13 +++++ 37 files changed, 192 insertions(+), 187 deletions(-) rename Dockerfile => build/package/Dockerfile (100%) rename build.bat => build/package/build.bat (100%) rename main.go => cmd/scraper/main.go (69%) rename docker-compose.old.yml => deployments/docker-compose.old.yml (100%) rename docker-compose.yaml => deployments/docker-compose.yaml (100%) delete mode 100644 e621/favorite.go delete mode 100644 e621/task.go delete mode 100644 e621/users.go rename {api => internal/api}/user.go (58%) rename {utils => internal/config}/config.go (97%) rename {database => internal/database}/neo4j/impl.go (92%) rename {database/neo4j/models => internal/database/neo4j/model}/tag.go (78%) rename {database => internal/database}/neo4j/post.go (100%) rename {database => internal/database}/neo4j/relationship.go (100%) rename {database => internal/database}/neo4j/source.go (100%) rename {database => internal/database}/neo4j/tag.go (83%) rename {database => internal/database}/neo4j/user.go (82%) rename {e621 => internal/e621}/client.go (68%) create mode 100644 internal/e621/favorites.go rename e621/request.go => internal/e621/scheduler/executor.go (73%) rename {e621 => internal/e621/scheduler}/scheduler.go (72%) rename e621/schedulertask.go => internal/e621/scheduler/task.go (64%) create mode 100644 internal/e621/task.go create mode 100644 internal/e621/user.go rename {services => internal/service}/manager.go (87%) rename {e621/models => pkg/e621/model}/post.go (99%) rename {e621/models => pkg/e621/model}/user.go (99%) rename utils/scheduler.go => pkg/e621/task.go (55%) rename {utils => pkg/error}/error.go (99%) rename database/logic.go => pkg/logic/database.go (66%) rename {utils => pkg/util}/list.go (98%) rename {utils => pkg/util/queue}/queue.go (98%) rename {utils => pkg/util/queue}/queue_test.go (99%) create mode 100644 pkg/util/queue/scheduler.go diff --git a/.gitea/workflows/build_check.yaml b/.gitea/workflows/build_check.yaml index 194da97..2bb31e1 100644 --- a/.gitea/workflows/build_check.yaml +++ b/.gitea/workflows/build_check.yaml @@ -2,7 +2,7 @@ name: Gitea Build Check run-name: ${{ gitea.actor }} is testing the build on: pull_request: - branches: ["master"] + branches: [ "master" ] jobs: Build: @@ -21,9 +21,9 @@ jobs: check-latest: true # optional # Used to specify whether caching is needed. Set to true, if you'd like to enable caching. cache: true # optional - + - name: Build run: go build -v ./... - + - name: Test run: go test -v ./... -json -coverprofile=coverage.out > test-report.out diff --git a/README.MD b/README.MD index 871f9e6..d192655 100644 --- a/README.MD +++ b/README.MD @@ -34,7 +34,8 @@ To install the project, follow these steps: ## Setting Up Environment Variables -The program requires certain environment variables to be set. +The program requires certain environment variables to be set. + ```plaintext E621_API_KEY= E621_USERNAME= @@ -93,7 +94,7 @@ The program supports the following databases. ### Neo4j -To improve performance, it is recommended to create indices on different nodes in Neo4j. +To improve performance, it is recommended to create indices on different nodes in Neo4j. Run the following commands to create the required indices: @@ -106,7 +107,7 @@ neo4j$ CREATE INDEX userIdIndex FOR (u:e621User) ON (u.e621ID); ### Memgraph: -To improve performance, it is recommended to create indices on different nodes in Memgraph. +To improve performance, it is recommended to create indices on different nodes in Memgraph. Run the following commands to create the required indices: diff --git a/Dockerfile b/build/package/Dockerfile similarity index 100% rename from Dockerfile rename to build/package/Dockerfile diff --git a/build.bat b/build/package/build.bat similarity index 100% rename from build.bat rename to build/package/build.bat diff --git a/main.go b/cmd/scraper/main.go similarity index 69% rename from main.go rename to cmd/scraper/main.go index c9dea53..fdd7686 100644 --- a/main.go +++ b/cmd/scraper/main.go @@ -2,11 +2,11 @@ package main import ( "context" - "e621_to_neo4j/api" - "e621_to_neo4j/database" - neo4j "e621_to_neo4j/database/neo4j" - "e621_to_neo4j/e621" - "e621_to_neo4j/utils" + "git.dragse.it/anthrove/e621-to-graph/internal/api" + "git.dragse.it/anthrove/e621-to-graph/internal/config" + "git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j" + "git.dragse.it/anthrove/e621-to-graph/internal/e621" + "git.dragse.it/anthrove/e621-to-graph/pkg/logic" log "github.com/sirupsen/logrus" "net/http" "os" @@ -30,11 +30,11 @@ func init() { func main() { - var graphConnection database.GraphConnection + var graphConnection logic.GraphConnection ctx := context.Background() // Loads Config - config, err := utils.LoadConfig() + config, err := config.LoadConfig() if err != nil { log.Println(err) } @@ -57,8 +57,8 @@ func main() { log.Printf("Im ready!") - // Register the UserHandler with the "/user" route - http.HandleFunc("/user", api.UserHandler(ctx, graphConnection, e621Client)) + // Register the ScapeUserFavourites with the "/user" route + http.HandleFunc("/user", api.ScapeUserFavourites(ctx, graphConnection, e621Client)) // Start the HTTP server err = http.ListenAndServe(":8080", nil) diff --git a/docker-compose.old.yml b/deployments/docker-compose.old.yml similarity index 100% rename from docker-compose.old.yml rename to deployments/docker-compose.old.yml diff --git a/docker-compose.yaml b/deployments/docker-compose.yaml similarity index 100% rename from docker-compose.yaml rename to deployments/docker-compose.yaml diff --git a/e621/favorite.go b/e621/favorite.go deleted file mode 100644 index 76f34bf..0000000 --- a/e621/favorite.go +++ /dev/null @@ -1,14 +0,0 @@ -package e621 - -import ( - "context" - "e621_to_neo4j/e621/models" - "fmt" -) - -// GetFavorites retrieves all favorites from the e621 API. -func (c *Client) GetFavorites(_ context.Context, user models.E621User, page int64) func() (models.PostResponseWrapper, error) { - URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page) - e621Task := NewE621ApiTask[models.PostResponseWrapper](URIPath) - return Schedule[models.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey) -} diff --git a/e621/task.go b/e621/task.go deleted file mode 100644 index 2f8397a..0000000 --- a/e621/task.go +++ /dev/null @@ -1,56 +0,0 @@ -package e621 - -import ( - "e621_to_neo4j/utils" - "encoding/json" - "net/http" -) - -type e621APITask[T utils.DataType] struct { - uri string -} - -func (e e621APITask[T]) UriPath() string { - return e.uri -} - -func (e e621APITask[T]) HandleStatusCode(statusCode int) utils.DataResponse[T] { - var err error - switch statusCode { - case 421: - err = utils.RateLimitReachedError{} - case 424: - err = utils.InvalidParametersError{} - case 520: - err = utils.OriginConnectionTimeOutError{} - case 522: - err = utils.OriginConnectionTimeOutError{} - case 524: - err = utils.OriginConnectionTimeOutError{} - case 525: - err = utils.SSLHandshakeFailedError{} - default: - err = utils.StatusCodesToError(statusCode) - } - return utils.DataResponse[T]{Error: err} -} - -func (e e621APITask[T]) HandleResponse(responseData *http.Response) utils.DataResponse[T] { - var data T - err := json.NewDecoder(responseData.Body).Decode(&data) - defer responseData.Body.Close() - if err != nil { - return utils.DataResponse[T]{Error: err} - } - return utils.DataResponse[T]{Data: data} -} - -func (e e621APITask[T]) HandleError(error error) utils.DataResponse[T] { - return utils.DataResponse[T]{Error: error} -} - -func NewE621ApiTask[T utils.DataType](uri string) utils.Task[T] { - return &e621APITask[T]{ - uri: uri, - } -} diff --git a/e621/users.go b/e621/users.go deleted file mode 100644 index b99f2a9..0000000 --- a/e621/users.go +++ /dev/null @@ -1,13 +0,0 @@ -package e621 - -import ( - "e621_to_neo4j/e621/models" - "fmt" -) - -// GetUserInfo retrieves the users information from e621 API. -func (c *Client) GetUserInfo(username string) func() (models.E621User, error) { - URIPath := fmt.Sprintf("users/%s.json", username) - e621Task := NewE621ApiTask[models.E621User](URIPath) - return Schedule[models.E621User](c.scheduler, e621Task, c.username, c.apiKey) -} diff --git a/go.mod b/go.mod index 4e98fff..8dbbecb 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module e621_to_neo4j +module git.dragse.it/anthrove/e621-to-graph go 1.20 diff --git a/api/user.go b/internal/api/user.go similarity index 58% rename from api/user.go rename to internal/api/user.go index 6d908de..def53c9 100644 --- a/api/user.go +++ b/internal/api/user.go @@ -2,16 +2,16 @@ package api import ( "context" - "e621_to_neo4j/database" - "e621_to_neo4j/e621" - "e621_to_neo4j/services" "fmt" + "git.dragse.it/anthrove/e621-to-graph/internal/e621" + "git.dragse.it/anthrove/e621-to-graph/internal/service" + "git.dragse.it/anthrove/e621-to-graph/pkg/logic" log "github.com/sirupsen/logrus" "net/http" ) -// UserHandler is the handler for the user API -func UserHandler(ctx context.Context, graphConnection database.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) { +// ScapeUserFavourites is the handler for the user API +func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) { return func(w http.ResponseWriter, r *http.Request) { log.Println("Request") if r.Method != http.MethodPost { @@ -28,7 +28,7 @@ func UserHandler(ctx context.Context, graphConnection database.GraphConnection, } // Perform further processing with the username - go services.ScrapeUser(ctx, graphConnection, *e621Client, username) + go service.ScrapeUser(ctx, graphConnection, *e621Client, username) // Send a response w.WriteHeader(http.StatusOK) diff --git a/utils/config.go b/internal/config/config.go similarity index 97% rename from utils/config.go rename to internal/config/config.go index ec2a727..787216b 100644 --- a/utils/config.go +++ b/internal/config/config.go @@ -1,4 +1,4 @@ -package utils +package config import ( "fmt" diff --git a/database/neo4j/impl.go b/internal/database/neo4j/impl.go similarity index 92% rename from database/neo4j/impl.go rename to internal/database/neo4j/impl.go index c2035fa..6be21e3 100644 --- a/database/neo4j/impl.go +++ b/internal/database/neo4j/impl.go @@ -2,8 +2,8 @@ package neo4j import ( "context" - "e621_to_neo4j/database" - "e621_to_neo4j/e621/models" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" + "git.dragse.it/anthrove/e621-to-graph/pkg/logic" "github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/neo4j/neo4j-go-driver/v5/neo4j/config" ) @@ -12,7 +12,7 @@ type neo4jConnection struct { driver neo4j.DriverWithContext } -func NewNeo4JConnection() database.GraphConnection { +func NewNeo4JConnection() logic.GraphConnection { return &neo4jConnection{} } @@ -44,7 +44,7 @@ func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) er return CreateSourceNode(ctx, c.driver, SourceURL) } -func (c *neo4jConnection) UploadUser(ctx context.Context, user models.E621User) error { +func (c *neo4jConnection) UploadUser(ctx context.Context, user model.E621User) error { return CreateUserNode(ctx, c.driver, user) } func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error { diff --git a/database/neo4j/models/tag.go b/internal/database/neo4j/model/tag.go similarity index 78% rename from database/neo4j/models/tag.go rename to internal/database/neo4j/model/tag.go index 72ff24a..9b966c1 100644 --- a/database/neo4j/models/tag.go +++ b/internal/database/neo4j/model/tag.go @@ -1,4 +1,4 @@ -package models +package model type DBTag struct { Tag string diff --git a/database/neo4j/post.go b/internal/database/neo4j/post.go similarity index 100% rename from database/neo4j/post.go rename to internal/database/neo4j/post.go diff --git a/database/neo4j/relationship.go b/internal/database/neo4j/relationship.go similarity index 100% rename from database/neo4j/relationship.go rename to internal/database/neo4j/relationship.go diff --git a/database/neo4j/source.go b/internal/database/neo4j/source.go similarity index 100% rename from database/neo4j/source.go rename to internal/database/neo4j/source.go diff --git a/database/neo4j/tag.go b/internal/database/neo4j/tag.go similarity index 83% rename from database/neo4j/tag.go rename to internal/database/neo4j/tag.go index a2c08a6..7a98aec 100644 --- a/database/neo4j/tag.go +++ b/internal/database/neo4j/tag.go @@ -2,7 +2,7 @@ package neo4j import ( "context" - "e621_to_neo4j/database/neo4j/models" + "git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j/model" "github.com/neo4j/neo4j-go-driver/v5/neo4j" ) @@ -24,9 +24,9 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str return nil } -func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string) (models.DBTag, bool, error) { +func GetTagNodeByName(ctx context.Context, driver neo4j.DriverWithContext, name string) (model.DBTag, bool, error) { - var tag models.DBTag + var tag model.DBTag query := ` MATCH (u:e621Tag {e621Tag: $name}) @@ -48,7 +48,7 @@ func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string e621Tag, _, _ := neo4j.GetRecordValue[string](record, "e621Tag") e621TagType, _, _ := neo4j.GetRecordValue[string](record, "e621TagType") - tag = models.DBTag{ + tag = model.DBTag{ Tag: e621Tag, TagType: e621TagType, } diff --git a/database/neo4j/user.go b/internal/database/neo4j/user.go similarity index 82% rename from database/neo4j/user.go rename to internal/database/neo4j/user.go index dab7a39..9541a2a 100644 --- a/database/neo4j/user.go +++ b/internal/database/neo4j/user.go @@ -2,11 +2,11 @@ package neo4j import ( "context" - "e621_to_neo4j/e621/models" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" "github.com/neo4j/neo4j-go-driver/v5/neo4j" ) -func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error { +func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user model.E621User) error { query := ` MERGE (u:e621User {e621ID: $id, e621Username: $name}) RETURN u diff --git a/e621/client.go b/internal/e621/client.go similarity index 68% rename from e621/client.go rename to internal/e621/client.go index 4528118..b1163d1 100644 --- a/e621/client.go +++ b/internal/e621/client.go @@ -1,21 +1,20 @@ package e621 -import "golang.org/x/time/rate" - -const ( - baseURL = "https://e621.net" +import ( + e621 "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler" + "golang.org/x/time/rate" ) // Client represents the e621 API client. type Client struct { apiKey string username string - scheduler *Scheduler + scheduler *e621.Scheduler } // NewClient creates a new e621 API client. func NewClient(apiKey string, username string) *Client { - scheduler := NewScheduler() + scheduler := e621.NewScheduler() scheduler.SetLimiter(rate.NewLimiter(1, 2)) return &Client{ apiKey: apiKey, diff --git a/internal/e621/favorites.go b/internal/e621/favorites.go new file mode 100644 index 0000000..379d77a --- /dev/null +++ b/internal/e621/favorites.go @@ -0,0 +1,15 @@ +package e621 + +import ( + "context" + "fmt" + "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" +) + +// GetFavorites retrieves all favorites from the e621 API. +func (c *Client) GetFavorites(_ context.Context, user model.E621User, page int64) func() (model.PostResponseWrapper, error) { + URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page) + e621Task := NewE621ApiTask[model.PostResponseWrapper](URIPath) + return scheduler.Schedule[model.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey) +} diff --git a/e621/request.go b/internal/e621/scheduler/executor.go similarity index 73% rename from e621/request.go rename to internal/e621/scheduler/executor.go index 147a9be..c6134d5 100644 --- a/e621/request.go +++ b/internal/e621/scheduler/executor.go @@ -1,17 +1,21 @@ -package e621 +package scheduler import ( - "e621_to_neo4j/utils" "fmt" + "git.dragse.it/anthrove/e621-to-graph/pkg/util/queue" log "github.com/sirupsen/logrus" "net/http" ) -func ExecuteGetAPIRequest(schedulerTask utils.SchedulerTask) { +const ( + BASEURL = "https://e621.net" +) + +func GetAPIRequest(schedulerTask queue.SchedulerTask) { var err error log.Debug("executing scheduler task") - url := fmt.Sprintf("%s/%s", baseURL, schedulerTask.UriPath()) + url := fmt.Sprintf("%s/%s", BASEURL, schedulerTask.UriPath()) req, err := http.NewRequest("GET", url, nil) if err != nil { schedulerTask.SendError(err) diff --git a/e621/scheduler.go b/internal/e621/scheduler/scheduler.go similarity index 72% rename from e621/scheduler.go rename to internal/e621/scheduler/scheduler.go index bb89fa6..ce96c91 100644 --- a/e621/scheduler.go +++ b/internal/e621/scheduler/scheduler.go @@ -1,28 +1,29 @@ -package e621 +package scheduler import ( "context" - "e621_to_neo4j/utils" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621" + "git.dragse.it/anthrove/e621-to-graph/pkg/util/queue" log "github.com/sirupsen/logrus" "golang.org/x/time/rate" ) type Scheduler struct { - queue utils.Queue + queue queue.Queue limiter *rate.Limiter } func NewScheduler() *Scheduler { scheduler := &Scheduler{ - queue: utils.NewQueue(), + queue: queue.NewQueue(), limiter: nil, } go scheduler.StartExecutionHandler() return scheduler } -func Schedule[T utils.DataType](s *Scheduler, t utils.Task[T], username string, apiKey string) func() (T, error) { - channel := make(chan utils.DataResponse[T]) +func Schedule[T e621.DataType](s *Scheduler, t e621.Task[T], username string, apiKey string) func() (T, error) { + channel := make(chan e621.DataResponse[T]) schedulerTask := NewSchedulerTaskImpl[T](t, channel, username, apiKey) log.Debug("Psuh task") err := s.queue.Push(schedulerTask) @@ -55,7 +56,7 @@ func (s *Scheduler) StartExecutionHandler() { if err != nil { continue } - ExecuteGetAPIRequest(task) + GetAPIRequest(task) } } diff --git a/e621/schedulertask.go b/internal/e621/scheduler/task.go similarity index 64% rename from e621/schedulertask.go rename to internal/e621/scheduler/task.go index 9cd6832..e158740 100644 --- a/e621/schedulertask.go +++ b/internal/e621/scheduler/task.go @@ -1,13 +1,14 @@ -package e621 +package scheduler import ( - "e621_to_neo4j/utils" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621" + "git.dragse.it/anthrove/e621-to-graph/pkg/util/queue" "net/http" ) -type schedulerTaskImpl[T utils.DataType] struct { - task utils.Task[T] - channel chan utils.DataResponse[T] +type schedulerTaskImpl[T e621.DataType] struct { + task e621.Task[T] + channel chan e621.DataResponse[T] username string apiKey string } @@ -16,7 +17,7 @@ func (s schedulerTaskImpl[T]) BasicAuth() (string, string) { return s.username, s.apiKey } -func NewSchedulerTaskImpl[T utils.DataType](task utils.Task[T], channel chan utils.DataResponse[T], username string, apiKey string) utils.SchedulerTask { +func NewSchedulerTaskImpl[T e621.DataType](task e621.Task[T], channel chan e621.DataResponse[T], username string, apiKey string) queue.SchedulerTask { return &schedulerTaskImpl[T]{ task: task, channel: channel, diff --git a/internal/e621/task.go b/internal/e621/task.go new file mode 100644 index 0000000..25b7719 --- /dev/null +++ b/internal/e621/task.go @@ -0,0 +1,57 @@ +package e621 + +import ( + "encoding/json" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621" + pgkError "git.dragse.it/anthrove/e621-to-graph/pkg/error" + "net/http" +) + +type e621APITask[T e621.DataType] struct { + uri string +} + +func NewE621ApiTask[T e621.DataType](uri string) e621.Task[T] { + return &e621APITask[T]{ + uri: uri, + } +} + +func (e e621APITask[T]) UriPath() string { + return e.uri +} + +func (e e621APITask[T]) HandleStatusCode(statusCode int) e621.DataResponse[T] { + var err error + switch statusCode { + case 421: + err = pgkError.RateLimitReachedError{} + case 424: + err = pgkError.InvalidParametersError{} + case 520: + err = pgkError.OriginConnectionTimeOutError{} + case 522: + err = pgkError.OriginConnectionTimeOutError{} + case 524: + err = pgkError.OriginConnectionTimeOutError{} + case 525: + err = pgkError.SSLHandshakeFailedError{} + default: + err = pgkError.StatusCodesToError(statusCode) + } + return e621.DataResponse[T]{Error: err} +} + +func (e e621APITask[T]) HandleResponse(responseData *http.Response) e621.DataResponse[T] { + var data T + err := json.NewDecoder(responseData.Body).Decode(&data) + defer responseData.Body.Close() + if err != nil { + return e621.DataResponse[T]{Error: err} + } + return e621.DataResponse[T]{Data: data} +} + +func (e e621APITask[T]) HandleError(error error) e621.DataResponse[T] { + return e621.DataResponse[T]{Error: error} +} diff --git a/internal/e621/user.go b/internal/e621/user.go new file mode 100644 index 0000000..8f71f35 --- /dev/null +++ b/internal/e621/user.go @@ -0,0 +1,14 @@ +package e621 + +import ( + "fmt" + "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" +) + +// GetUserInfo retrieves the users information from e621 API. +func (c *Client) GetUserInfo(username string) func() (model.E621User, error) { + URIPath := fmt.Sprintf("users/%s.json", username) + e621Task := NewE621ApiTask[model.E621User](URIPath) + return scheduler.Schedule[model.E621User](c.scheduler, e621Task, c.username, c.apiKey) +} diff --git a/services/manager.go b/internal/service/manager.go similarity index 87% rename from services/manager.go rename to internal/service/manager.go index 84622ed..2d04cb6 100644 --- a/services/manager.go +++ b/internal/service/manager.go @@ -1,19 +1,19 @@ -package services +package service import ( "context" - "e621_to_neo4j/database" - "e621_to_neo4j/e621" - "e621_to_neo4j/e621/models" - "e621_to_neo4j/utils" + "git.dragse.it/anthrove/e621-to-graph/internal/e621" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" + "git.dragse.it/anthrove/e621-to-graph/pkg/logic" + "git.dragse.it/anthrove/e621-to-graph/pkg/util" log "github.com/sirupsen/logrus" "time" ) -func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e621Client e621.Client, username string) error { +func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621Client e621.Client, username string) error { var err error var page int64 = 1 - var allFavorites []models.Post + var allFavorites []model.Post AwaitE621User := e621Client.GetUserInfo(username) e621User, err := AwaitE621User() @@ -167,7 +167,7 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e } // uploadNodes uploads the post to the database and creates the nodes -func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { +func uploadNodes(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { uniqueGeneralTags := make([]string, 0) uniqueCharacterTags := make([]string, 0) @@ -184,10 +184,10 @@ func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, allCopyrightTags = append(allCopyrightTags, post.Tags.Copyright...) allArtistTags = append(allArtistTags, post.Tags.Artist...) - uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) - uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) - uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) - uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) + uniqueGeneralTags = util.UniqueNonEmptyElementsOf(allGeneralTags) + uniqueCharacterTags = util.UniqueNonEmptyElementsOf(allCharacterTags) + uniqueCopyrightTags = util.UniqueNonEmptyElementsOf(allCopyrightTags) + uniqueArtistTags = util.UniqueNonEmptyElementsOf(allArtistTags) err := graphConnection.UploadPost(ctx, post.ID) if err != nil { @@ -233,7 +233,7 @@ func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, } // uploadPostToUserRelationship creates a relationship between the user and the post -func uploadPostToUserRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post, e621User models.E621User) error { +func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.E621User) error { err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID) if err != nil { return err @@ -243,7 +243,7 @@ func uploadPostToUserRelationship(ctx context.Context, graphConnection database. } // uploadSourceTagRelationship creates a relationship between the post and the source -func uploadSourceTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { +func uploadSourceTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, source := range post.Sources { err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source) if err != nil { @@ -256,7 +256,7 @@ func uploadSourceTagRelationship(ctx context.Context, graphConnection database.G } // uploadGeneralTagRelationship creates a relationship between the post and the general tag -func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { +func uploadGeneralTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, generalTag := range post.Tags.General { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag) if err != nil { @@ -269,7 +269,7 @@ func uploadGeneralTagRelationship(ctx context.Context, graphConnection database. } // uploadCharacterTagtRelationship creates a relationship between the post and the character tag -func uploadCharacterTagtRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { +func uploadCharacterTagtRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, characterTag := range post.Tags.Character { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag) if err != nil { @@ -282,7 +282,7 @@ func uploadCharacterTagtRelationship(ctx context.Context, graphConnection databa } // uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag -func uploadCopyrightTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { +func uploadCopyrightTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, copyrightTag := range post.Tags.Copyright { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag) if err != nil { @@ -295,7 +295,7 @@ func uploadCopyrightTagRelationship(ctx context.Context, graphConnection databas } // uploadArtistTagRelationship creates a relationship between the post and the artist tag -func uploadArtistTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { +func uploadArtistTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, artistTag := range post.Tags.Artist { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag) if err != nil { diff --git a/e621/models/post.go b/pkg/e621/model/post.go similarity index 99% rename from e621/models/post.go rename to pkg/e621/model/post.go index dcc7c3f..ae307db 100644 --- a/e621/models/post.go +++ b/pkg/e621/model/post.go @@ -1,4 +1,4 @@ -package models +package model import "encoding/json" diff --git a/e621/models/user.go b/pkg/e621/model/user.go similarity index 99% rename from e621/models/user.go rename to pkg/e621/model/user.go index 7018463..5113500 100644 --- a/e621/models/user.go +++ b/pkg/e621/model/user.go @@ -1,4 +1,4 @@ -package models +package model import "encoding/json" diff --git a/utils/scheduler.go b/pkg/e621/task.go similarity index 55% rename from utils/scheduler.go rename to pkg/e621/task.go index b59692d..7e69ec5 100644 --- a/utils/scheduler.go +++ b/pkg/e621/task.go @@ -1,7 +1,7 @@ -package utils +package e621 import ( - "e621_to_neo4j/e621/models" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" "net/http" ) @@ -11,7 +11,7 @@ type DataResponse[T DataType] struct { } type DataType interface { - models.E621User | models.PostResponseWrapper + model.E621User | model.PostResponseWrapper } type Task[T DataType] interface { @@ -20,11 +20,3 @@ type Task[T DataType] interface { HandleStatusCode(statusCode int) DataResponse[T] HandleResponse(responseData *http.Response) DataResponse[T] } - -type SchedulerTask interface { - UriPath() string - SendStatusCode(statusCode int) - SendError(err error) - SendResponse(response *http.Response) - BasicAuth() (string, string) -} diff --git a/utils/error.go b/pkg/error/error.go similarity index 99% rename from utils/error.go rename to pkg/error/error.go index 784f81c..1e17d37 100644 --- a/utils/error.go +++ b/pkg/error/error.go @@ -1,4 +1,4 @@ -package utils +package error import "fmt" diff --git a/database/logic.go b/pkg/logic/database.go similarity index 66% rename from database/logic.go rename to pkg/logic/database.go index 6d75eb7..4b3fc10 100644 --- a/database/logic.go +++ b/pkg/logic/database.go @@ -1,22 +1,13 @@ -package database +package logic import ( "context" - "e621_to_neo4j/e621/models" + "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" ) -// [X:Neo4J] Connection -// [X:Neo4J] Upload User -// [X:Neo4J] Upload Source -// [X:Neo4J] Upload Post -// [X:Neo4J] Upload Tag -// [X:Neo4J] Relationship Post->Tag -// [X:Neo4J] Relationship Post->Source -// [X:Neo4J] Relationship User->Post - type GraphConnection interface { Connect(ctx context.Context, endpoint string, username string, password string) error - UploadUser(ctx context.Context, user models.E621User) error + UploadUser(ctx context.Context, user model.E621User) error UploadSource(ctx context.Context, SourceURL string) error UploadPost(ctx context.Context, e621ID int64) error UploadTag(ctx context.Context, name string, tagType string) error diff --git a/utils/list.go b/pkg/util/list.go similarity index 98% rename from utils/list.go rename to pkg/util/list.go index a4682f4..3435940 100644 --- a/utils/list.go +++ b/pkg/util/list.go @@ -1,4 +1,4 @@ -package utils +package util // UniqueNonEmptyElementsOf returns a new slice containing unique non-empty elements from the input slice. // It removes duplicate elements and empty strings while preserving the order of appearance. diff --git a/utils/queue.go b/pkg/util/queue/queue.go similarity index 98% rename from utils/queue.go rename to pkg/util/queue/queue.go index 7b6f4f0..37e7c1d 100644 --- a/utils/queue.go +++ b/pkg/util/queue/queue.go @@ -1,4 +1,4 @@ -package utils +package queue import ( "errors" diff --git a/utils/queue_test.go b/pkg/util/queue/queue_test.go similarity index 99% rename from utils/queue_test.go rename to pkg/util/queue/queue_test.go index aa323bf..8db29c1 100644 --- a/utils/queue_test.go +++ b/pkg/util/queue/queue_test.go @@ -1,4 +1,4 @@ -package utils +package queue import ( "net/http" diff --git a/pkg/util/queue/scheduler.go b/pkg/util/queue/scheduler.go new file mode 100644 index 0000000..122e9c4 --- /dev/null +++ b/pkg/util/queue/scheduler.go @@ -0,0 +1,13 @@ +package queue + +import ( + "net/http" +) + +type SchedulerTask interface { + UriPath() string + SendStatusCode(statusCode int) + SendError(err error) + SendResponse(response *http.Response) + BasicAuth() (string, string) +}