refactor_folder_structure_#11 (#14)

As mentioned in Issue #11, the folder structure got an overall as some file names

Co-authored-by: Fenpaws <soxx@fenpa.ws>
Reviewed-on: anthrove/e621-to-graph#14
Reviewed-by: Lennard Brinkhaus <lennard.brinkhaus@noreply.localhost>
Co-authored-by: SoXX <fenpaws@noreply.localhost>
Co-committed-by: SoXX <fenpaws@noreply.localhost>
This commit is contained in:
SoXX 2023-07-17 10:57:23 +00:00 committed by Lennard Brinkhaus
parent 31bd0cf639
commit 802764092e
37 changed files with 192 additions and 187 deletions

View File

@ -2,7 +2,7 @@ name: Gitea Build Check
run-name: ${{ gitea.actor }} is testing the build run-name: ${{ gitea.actor }} is testing the build
on: on:
pull_request: pull_request:
branches: ["master"] branches: [ "master" ]
jobs: jobs:
Build: Build:
@ -21,9 +21,9 @@ jobs:
check-latest: true # optional check-latest: true # optional
# Used to specify whether caching is needed. Set to true, if you'd like to enable caching. # Used to specify whether caching is needed. Set to true, if you'd like to enable caching.
cache: true # optional cache: true # optional
- name: Build - name: Build
run: go build -v ./... run: go build -v ./...
- name: Test - name: Test
run: go test -v ./... -json -coverprofile=coverage.out > test-report.out run: go test -v ./... -json -coverprofile=coverage.out > test-report.out

View File

@ -34,7 +34,8 @@ To install the project, follow these steps:
## Setting Up Environment Variables ## Setting Up Environment Variables
The program requires certain environment variables to be set. The program requires certain environment variables to be set.
```plaintext ```plaintext
E621_API_KEY= E621_API_KEY=
E621_USERNAME= E621_USERNAME=
@ -93,7 +94,7 @@ The program supports the following databases.
### Neo4j ### Neo4j
To improve performance, it is recommended to create indices on different nodes in Neo4j. To improve performance, it is recommended to create indices on different nodes in Neo4j.
Run the following commands to create the required indices: Run the following commands to create the required indices:
@ -106,7 +107,7 @@ neo4j$ CREATE INDEX userIdIndex FOR (u:e621User) ON (u.e621ID);
### Memgraph: ### Memgraph:
To improve performance, it is recommended to create indices on different nodes in Memgraph. To improve performance, it is recommended to create indices on different nodes in Memgraph.
Run the following commands to create the required indices: Run the following commands to create the required indices:

View File

@ -2,11 +2,11 @@ package main
import ( import (
"context" "context"
"e621_to_neo4j/api" "git.dragse.it/anthrove/e621-to-graph/internal/api"
"e621_to_neo4j/database" "git.dragse.it/anthrove/e621-to-graph/internal/config"
neo4j "e621_to_neo4j/database/neo4j" "git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j"
"e621_to_neo4j/e621" "git.dragse.it/anthrove/e621-to-graph/internal/e621"
"e621_to_neo4j/utils" "git.dragse.it/anthrove/e621-to-graph/pkg/logic"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"net/http" "net/http"
"os" "os"
@ -30,11 +30,11 @@ func init() {
func main() { func main() {
var graphConnection database.GraphConnection var graphConnection logic.GraphConnection
ctx := context.Background() ctx := context.Background()
// Loads Config // Loads Config
config, err := utils.LoadConfig() config, err := config.LoadConfig()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
} }
@ -57,8 +57,8 @@ func main() {
log.Printf("Im ready!") log.Printf("Im ready!")
// Register the UserHandler with the "/user" route // Register the ScapeUserFavourites with the "/user" route
http.HandleFunc("/user", api.UserHandler(ctx, graphConnection, e621Client)) http.HandleFunc("/user", api.ScapeUserFavourites(ctx, graphConnection, e621Client))
// Start the HTTP server // Start the HTTP server
err = http.ListenAndServe(":8080", nil) err = http.ListenAndServe(":8080", nil)

View File

@ -1,14 +0,0 @@
package e621
import (
"context"
"e621_to_neo4j/e621/models"
"fmt"
)
// GetFavorites retrieves all favorites from the e621 API.
func (c *Client) GetFavorites(_ context.Context, user models.E621User, page int64) func() (models.PostResponseWrapper, error) {
URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page)
e621Task := NewE621ApiTask[models.PostResponseWrapper](URIPath)
return Schedule[models.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey)
}

View File

@ -1,56 +0,0 @@
package e621
import (
"e621_to_neo4j/utils"
"encoding/json"
"net/http"
)
type e621APITask[T utils.DataType] struct {
uri string
}
func (e e621APITask[T]) UriPath() string {
return e.uri
}
func (e e621APITask[T]) HandleStatusCode(statusCode int) utils.DataResponse[T] {
var err error
switch statusCode {
case 421:
err = utils.RateLimitReachedError{}
case 424:
err = utils.InvalidParametersError{}
case 520:
err = utils.OriginConnectionTimeOutError{}
case 522:
err = utils.OriginConnectionTimeOutError{}
case 524:
err = utils.OriginConnectionTimeOutError{}
case 525:
err = utils.SSLHandshakeFailedError{}
default:
err = utils.StatusCodesToError(statusCode)
}
return utils.DataResponse[T]{Error: err}
}
func (e e621APITask[T]) HandleResponse(responseData *http.Response) utils.DataResponse[T] {
var data T
err := json.NewDecoder(responseData.Body).Decode(&data)
defer responseData.Body.Close()
if err != nil {
return utils.DataResponse[T]{Error: err}
}
return utils.DataResponse[T]{Data: data}
}
func (e e621APITask[T]) HandleError(error error) utils.DataResponse[T] {
return utils.DataResponse[T]{Error: error}
}
func NewE621ApiTask[T utils.DataType](uri string) utils.Task[T] {
return &e621APITask[T]{
uri: uri,
}
}

View File

@ -1,13 +0,0 @@
package e621
import (
"e621_to_neo4j/e621/models"
"fmt"
)
// GetUserInfo retrieves the users information from e621 API.
func (c *Client) GetUserInfo(username string) func() (models.E621User, error) {
URIPath := fmt.Sprintf("users/%s.json", username)
e621Task := NewE621ApiTask[models.E621User](URIPath)
return Schedule[models.E621User](c.scheduler, e621Task, c.username, c.apiKey)
}

2
go.mod
View File

@ -1,4 +1,4 @@
module e621_to_neo4j module git.dragse.it/anthrove/e621-to-graph
go 1.20 go 1.20

View File

@ -2,16 +2,16 @@ package api
import ( import (
"context" "context"
"e621_to_neo4j/database"
"e621_to_neo4j/e621"
"e621_to_neo4j/services"
"fmt" "fmt"
"git.dragse.it/anthrove/e621-to-graph/internal/e621"
"git.dragse.it/anthrove/e621-to-graph/internal/service"
"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"net/http" "net/http"
) )
// UserHandler is the handler for the user API // ScapeUserFavourites is the handler for the user API
func UserHandler(ctx context.Context, graphConnection database.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) { func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
log.Println("Request") log.Println("Request")
if r.Method != http.MethodPost { if r.Method != http.MethodPost {
@ -28,7 +28,7 @@ func UserHandler(ctx context.Context, graphConnection database.GraphConnection,
} }
// Perform further processing with the username // Perform further processing with the username
go services.ScrapeUser(ctx, graphConnection, *e621Client, username) go service.ScrapeUser(ctx, graphConnection, *e621Client, username)
// Send a response // Send a response
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)

View File

@ -1,4 +1,4 @@
package utils package config
import ( import (
"fmt" "fmt"

View File

@ -2,8 +2,8 @@ package neo4j
import ( import (
"context" "context"
"e621_to_neo4j/database" "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
"e621_to_neo4j/e621/models" "git.dragse.it/anthrove/e621-to-graph/pkg/logic"
"github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/neo4j/neo4j-go-driver/v5/neo4j"
"github.com/neo4j/neo4j-go-driver/v5/neo4j/config" "github.com/neo4j/neo4j-go-driver/v5/neo4j/config"
) )
@ -12,7 +12,7 @@ type neo4jConnection struct {
driver neo4j.DriverWithContext driver neo4j.DriverWithContext
} }
func NewNeo4JConnection() database.GraphConnection { func NewNeo4JConnection() logic.GraphConnection {
return &neo4jConnection{} return &neo4jConnection{}
} }
@ -44,7 +44,7 @@ func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) er
return CreateSourceNode(ctx, c.driver, SourceURL) return CreateSourceNode(ctx, c.driver, SourceURL)
} }
func (c *neo4jConnection) UploadUser(ctx context.Context, user models.E621User) error { func (c *neo4jConnection) UploadUser(ctx context.Context, user model.E621User) error {
return CreateUserNode(ctx, c.driver, user) return CreateUserNode(ctx, c.driver, user)
} }
func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error { func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error {

View File

@ -1,4 +1,4 @@
package models package model
type DBTag struct { type DBTag struct {
Tag string Tag string

View File

@ -2,7 +2,7 @@ package neo4j
import ( import (
"context" "context"
"e621_to_neo4j/database/neo4j/models" "git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j/model"
"github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/neo4j/neo4j-go-driver/v5/neo4j"
) )
@ -24,9 +24,9 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str
return nil return nil
} }
func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string) (models.DBTag, bool, error) { func GetTagNodeByName(ctx context.Context, driver neo4j.DriverWithContext, name string) (model.DBTag, bool, error) {
var tag models.DBTag var tag model.DBTag
query := ` query := `
MATCH (u:e621Tag {e621Tag: $name}) MATCH (u:e621Tag {e621Tag: $name})
@ -48,7 +48,7 @@ func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string
e621Tag, _, _ := neo4j.GetRecordValue[string](record, "e621Tag") e621Tag, _, _ := neo4j.GetRecordValue[string](record, "e621Tag")
e621TagType, _, _ := neo4j.GetRecordValue[string](record, "e621TagType") e621TagType, _, _ := neo4j.GetRecordValue[string](record, "e621TagType")
tag = models.DBTag{ tag = model.DBTag{
Tag: e621Tag, Tag: e621Tag,
TagType: e621TagType, TagType: e621TagType,
} }

View File

@ -2,11 +2,11 @@ package neo4j
import ( import (
"context" "context"
"e621_to_neo4j/e621/models" "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
"github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/neo4j/neo4j-go-driver/v5/neo4j"
) )
func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error { func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user model.E621User) error {
query := ` query := `
MERGE (u:e621User {e621ID: $id, e621Username: $name}) MERGE (u:e621User {e621ID: $id, e621Username: $name})
RETURN u RETURN u

View File

@ -1,21 +1,20 @@
package e621 package e621
import "golang.org/x/time/rate" import (
e621 "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
const ( "golang.org/x/time/rate"
baseURL = "https://e621.net"
) )
// Client represents the e621 API client. // Client represents the e621 API client.
type Client struct { type Client struct {
apiKey string apiKey string
username string username string
scheduler *Scheduler scheduler *e621.Scheduler
} }
// NewClient creates a new e621 API client. // NewClient creates a new e621 API client.
func NewClient(apiKey string, username string) *Client { func NewClient(apiKey string, username string) *Client {
scheduler := NewScheduler() scheduler := e621.NewScheduler()
scheduler.SetLimiter(rate.NewLimiter(1, 2)) scheduler.SetLimiter(rate.NewLimiter(1, 2))
return &Client{ return &Client{
apiKey: apiKey, apiKey: apiKey,

View File

@ -0,0 +1,15 @@
package e621
import (
"context"
"fmt"
"git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
)
// GetFavorites retrieves all favorites from the e621 API.
func (c *Client) GetFavorites(_ context.Context, user model.E621User, page int64) func() (model.PostResponseWrapper, error) {
URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page)
e621Task := NewE621ApiTask[model.PostResponseWrapper](URIPath)
return scheduler.Schedule[model.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey)
}

View File

@ -1,17 +1,21 @@
package e621 package scheduler
import ( import (
"e621_to_neo4j/utils"
"fmt" "fmt"
"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"net/http" "net/http"
) )
func ExecuteGetAPIRequest(schedulerTask utils.SchedulerTask) { const (
BASEURL = "https://e621.net"
)
func GetAPIRequest(schedulerTask queue.SchedulerTask) {
var err error var err error
log.Debug("executing scheduler task") log.Debug("executing scheduler task")
url := fmt.Sprintf("%s/%s", baseURL, schedulerTask.UriPath()) url := fmt.Sprintf("%s/%s", BASEURL, schedulerTask.UriPath())
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {
schedulerTask.SendError(err) schedulerTask.SendError(err)

View File

@ -1,28 +1,29 @@
package e621 package scheduler
import ( import (
"context" "context"
"e621_to_neo4j/utils" "git.dragse.it/anthrove/e621-to-graph/pkg/e621"
"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"golang.org/x/time/rate" "golang.org/x/time/rate"
) )
type Scheduler struct { type Scheduler struct {
queue utils.Queue queue queue.Queue
limiter *rate.Limiter limiter *rate.Limiter
} }
func NewScheduler() *Scheduler { func NewScheduler() *Scheduler {
scheduler := &Scheduler{ scheduler := &Scheduler{
queue: utils.NewQueue(), queue: queue.NewQueue(),
limiter: nil, limiter: nil,
} }
go scheduler.StartExecutionHandler() go scheduler.StartExecutionHandler()
return scheduler return scheduler
} }
func Schedule[T utils.DataType](s *Scheduler, t utils.Task[T], username string, apiKey string) func() (T, error) { func Schedule[T e621.DataType](s *Scheduler, t e621.Task[T], username string, apiKey string) func() (T, error) {
channel := make(chan utils.DataResponse[T]) channel := make(chan e621.DataResponse[T])
schedulerTask := NewSchedulerTaskImpl[T](t, channel, username, apiKey) schedulerTask := NewSchedulerTaskImpl[T](t, channel, username, apiKey)
log.Debug("Psuh task") log.Debug("Psuh task")
err := s.queue.Push(schedulerTask) err := s.queue.Push(schedulerTask)
@ -55,7 +56,7 @@ func (s *Scheduler) StartExecutionHandler() {
if err != nil { if err != nil {
continue continue
} }
ExecuteGetAPIRequest(task) GetAPIRequest(task)
} }
} }

View File

@ -1,13 +1,14 @@
package e621 package scheduler
import ( import (
"e621_to_neo4j/utils" "git.dragse.it/anthrove/e621-to-graph/pkg/e621"
"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
"net/http" "net/http"
) )
type schedulerTaskImpl[T utils.DataType] struct { type schedulerTaskImpl[T e621.DataType] struct {
task utils.Task[T] task e621.Task[T]
channel chan utils.DataResponse[T] channel chan e621.DataResponse[T]
username string username string
apiKey string apiKey string
} }
@ -16,7 +17,7 @@ func (s schedulerTaskImpl[T]) BasicAuth() (string, string) {
return s.username, s.apiKey return s.username, s.apiKey
} }
func NewSchedulerTaskImpl[T utils.DataType](task utils.Task[T], channel chan utils.DataResponse[T], username string, apiKey string) utils.SchedulerTask { func NewSchedulerTaskImpl[T e621.DataType](task e621.Task[T], channel chan e621.DataResponse[T], username string, apiKey string) queue.SchedulerTask {
return &schedulerTaskImpl[T]{ return &schedulerTaskImpl[T]{
task: task, task: task,
channel: channel, channel: channel,

57
internal/e621/task.go Normal file
View File

@ -0,0 +1,57 @@
package e621
import (
"encoding/json"
"git.dragse.it/anthrove/e621-to-graph/pkg/e621"
pgkError "git.dragse.it/anthrove/e621-to-graph/pkg/error"
"net/http"
)
type e621APITask[T e621.DataType] struct {
uri string
}
func NewE621ApiTask[T e621.DataType](uri string) e621.Task[T] {
return &e621APITask[T]{
uri: uri,
}
}
func (e e621APITask[T]) UriPath() string {
return e.uri
}
func (e e621APITask[T]) HandleStatusCode(statusCode int) e621.DataResponse[T] {
var err error
switch statusCode {
case 421:
err = pgkError.RateLimitReachedError{}
case 424:
err = pgkError.InvalidParametersError{}
case 520:
err = pgkError.OriginConnectionTimeOutError{}
case 522:
err = pgkError.OriginConnectionTimeOutError{}
case 524:
err = pgkError.OriginConnectionTimeOutError{}
case 525:
err = pgkError.SSLHandshakeFailedError{}
default:
err = pgkError.StatusCodesToError(statusCode)
}
return e621.DataResponse[T]{Error: err}
}
func (e e621APITask[T]) HandleResponse(responseData *http.Response) e621.DataResponse[T] {
var data T
err := json.NewDecoder(responseData.Body).Decode(&data)
defer responseData.Body.Close()
if err != nil {
return e621.DataResponse[T]{Error: err}
}
return e621.DataResponse[T]{Data: data}
}
func (e e621APITask[T]) HandleError(error error) e621.DataResponse[T] {
return e621.DataResponse[T]{Error: error}
}

14
internal/e621/user.go Normal file
View File

@ -0,0 +1,14 @@
package e621
import (
"fmt"
"git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
)
// GetUserInfo retrieves the users information from e621 API.
func (c *Client) GetUserInfo(username string) func() (model.E621User, error) {
URIPath := fmt.Sprintf("users/%s.json", username)
e621Task := NewE621ApiTask[model.E621User](URIPath)
return scheduler.Schedule[model.E621User](c.scheduler, e621Task, c.username, c.apiKey)
}

View File

@ -1,19 +1,19 @@
package services package service
import ( import (
"context" "context"
"e621_to_neo4j/database" "git.dragse.it/anthrove/e621-to-graph/internal/e621"
"e621_to_neo4j/e621" "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
"e621_to_neo4j/e621/models" "git.dragse.it/anthrove/e621-to-graph/pkg/logic"
"e621_to_neo4j/utils" "git.dragse.it/anthrove/e621-to-graph/pkg/util"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"time" "time"
) )
func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e621Client e621.Client, username string) error { func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621Client e621.Client, username string) error {
var err error var err error
var page int64 = 1 var page int64 = 1
var allFavorites []models.Post var allFavorites []model.Post
AwaitE621User := e621Client.GetUserInfo(username) AwaitE621User := e621Client.GetUserInfo(username)
e621User, err := AwaitE621User() e621User, err := AwaitE621User()
@ -167,7 +167,7 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
} }
// uploadNodes uploads the post to the database and creates the nodes // uploadNodes uploads the post to the database and creates the nodes
func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { func uploadNodes(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
uniqueGeneralTags := make([]string, 0) uniqueGeneralTags := make([]string, 0)
uniqueCharacterTags := make([]string, 0) uniqueCharacterTags := make([]string, 0)
@ -184,10 +184,10 @@ func uploadNodes(ctx context.Context, graphConnection database.GraphConnection,
allCopyrightTags = append(allCopyrightTags, post.Tags.Copyright...) allCopyrightTags = append(allCopyrightTags, post.Tags.Copyright...)
allArtistTags = append(allArtistTags, post.Tags.Artist...) allArtistTags = append(allArtistTags, post.Tags.Artist...)
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags) uniqueGeneralTags = util.UniqueNonEmptyElementsOf(allGeneralTags)
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags) uniqueCharacterTags = util.UniqueNonEmptyElementsOf(allCharacterTags)
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags) uniqueCopyrightTags = util.UniqueNonEmptyElementsOf(allCopyrightTags)
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags) uniqueArtistTags = util.UniqueNonEmptyElementsOf(allArtistTags)
err := graphConnection.UploadPost(ctx, post.ID) err := graphConnection.UploadPost(ctx, post.ID)
if err != nil { if err != nil {
@ -233,7 +233,7 @@ func uploadNodes(ctx context.Context, graphConnection database.GraphConnection,
} }
// uploadPostToUserRelationship creates a relationship between the user and the post // uploadPostToUserRelationship creates a relationship between the user and the post
func uploadPostToUserRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post, e621User models.E621User) error { func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.E621User) error {
err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID) err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID)
if err != nil { if err != nil {
return err return err
@ -243,7 +243,7 @@ func uploadPostToUserRelationship(ctx context.Context, graphConnection database.
} }
// uploadSourceTagRelationship creates a relationship between the post and the source // uploadSourceTagRelationship creates a relationship between the post and the source
func uploadSourceTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { func uploadSourceTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
for _, source := range post.Sources { for _, source := range post.Sources {
err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source) err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source)
if err != nil { if err != nil {
@ -256,7 +256,7 @@ func uploadSourceTagRelationship(ctx context.Context, graphConnection database.G
} }
// uploadGeneralTagRelationship creates a relationship between the post and the general tag // uploadGeneralTagRelationship creates a relationship between the post and the general tag
func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { func uploadGeneralTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
for _, generalTag := range post.Tags.General { for _, generalTag := range post.Tags.General {
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag) err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag)
if err != nil { if err != nil {
@ -269,7 +269,7 @@ func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.
} }
// uploadCharacterTagtRelationship creates a relationship between the post and the character tag // uploadCharacterTagtRelationship creates a relationship between the post and the character tag
func uploadCharacterTagtRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { func uploadCharacterTagtRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
for _, characterTag := range post.Tags.Character { for _, characterTag := range post.Tags.Character {
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag) err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag)
if err != nil { if err != nil {
@ -282,7 +282,7 @@ func uploadCharacterTagtRelationship(ctx context.Context, graphConnection databa
} }
// uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag // uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag
func uploadCopyrightTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { func uploadCopyrightTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
for _, copyrightTag := range post.Tags.Copyright { for _, copyrightTag := range post.Tags.Copyright {
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag) err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag)
if err != nil { if err != nil {
@ -295,7 +295,7 @@ func uploadCopyrightTagRelationship(ctx context.Context, graphConnection databas
} }
// uploadArtistTagRelationship creates a relationship between the post and the artist tag // uploadArtistTagRelationship creates a relationship between the post and the artist tag
func uploadArtistTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error { func uploadArtistTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
for _, artistTag := range post.Tags.Artist { for _, artistTag := range post.Tags.Artist {
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag) err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag)
if err != nil { if err != nil {

View File

@ -1,4 +1,4 @@
package models package model
import "encoding/json" import "encoding/json"

View File

@ -1,4 +1,4 @@
package models package model
import "encoding/json" import "encoding/json"

View File

@ -1,7 +1,7 @@
package utils package e621
import ( import (
"e621_to_neo4j/e621/models" "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
"net/http" "net/http"
) )
@ -11,7 +11,7 @@ type DataResponse[T DataType] struct {
} }
type DataType interface { type DataType interface {
models.E621User | models.PostResponseWrapper model.E621User | model.PostResponseWrapper
} }
type Task[T DataType] interface { type Task[T DataType] interface {
@ -20,11 +20,3 @@ type Task[T DataType] interface {
HandleStatusCode(statusCode int) DataResponse[T] HandleStatusCode(statusCode int) DataResponse[T]
HandleResponse(responseData *http.Response) DataResponse[T] HandleResponse(responseData *http.Response) DataResponse[T]
} }
type SchedulerTask interface {
UriPath() string
SendStatusCode(statusCode int)
SendError(err error)
SendResponse(response *http.Response)
BasicAuth() (string, string)
}

View File

@ -1,4 +1,4 @@
package utils package error
import "fmt" import "fmt"

View File

@ -1,22 +1,13 @@
package database package logic
import ( import (
"context" "context"
"e621_to_neo4j/e621/models" "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
) )
// [X:Neo4J] Connection
// [X:Neo4J] Upload User
// [X:Neo4J] Upload Source
// [X:Neo4J] Upload Post
// [X:Neo4J] Upload Tag
// [X:Neo4J] Relationship Post->Tag
// [X:Neo4J] Relationship Post->Source
// [X:Neo4J] Relationship User->Post
type GraphConnection interface { type GraphConnection interface {
Connect(ctx context.Context, endpoint string, username string, password string) error Connect(ctx context.Context, endpoint string, username string, password string) error
UploadUser(ctx context.Context, user models.E621User) error UploadUser(ctx context.Context, user model.E621User) error
UploadSource(ctx context.Context, SourceURL string) error UploadSource(ctx context.Context, SourceURL string) error
UploadPost(ctx context.Context, e621ID int64) error UploadPost(ctx context.Context, e621ID int64) error
UploadTag(ctx context.Context, name string, tagType string) error UploadTag(ctx context.Context, name string, tagType string) error

View File

@ -1,4 +1,4 @@
package utils package util
// UniqueNonEmptyElementsOf returns a new slice containing unique non-empty elements from the input slice. // UniqueNonEmptyElementsOf returns a new slice containing unique non-empty elements from the input slice.
// It removes duplicate elements and empty strings while preserving the order of appearance. // It removes duplicate elements and empty strings while preserving the order of appearance.

View File

@ -1,4 +1,4 @@
package utils package queue
import ( import (
"errors" "errors"

View File

@ -1,4 +1,4 @@
package utils package queue
import ( import (
"net/http" "net/http"

View File

@ -0,0 +1,13 @@
package queue
import (
"net/http"
)
type SchedulerTask interface {
UriPath() string
SendStatusCode(statusCode int)
SendError(err error)
SendResponse(response *http.Response)
BasicAuth() (string, string)
}