refactor_folder_structure_#11 (#14)
As mentioned in Issue #11, the folder structure got an overall as some file names Co-authored-by: Fenpaws <soxx@fenpa.ws> Reviewed-on: anthrove/e621-to-graph#14 Reviewed-by: Lennard Brinkhaus <lennard.brinkhaus@noreply.localhost> Co-authored-by: SoXX <fenpaws@noreply.localhost> Co-committed-by: SoXX <fenpaws@noreply.localhost>
This commit is contained in:
parent
31bd0cf639
commit
802764092e
@ -2,7 +2,7 @@ name: Gitea Build Check
|
||||
run-name: ${{ gitea.actor }} is testing the build
|
||||
on:
|
||||
pull_request:
|
||||
branches: ["master"]
|
||||
branches: [ "master" ]
|
||||
|
||||
jobs:
|
||||
Build:
|
||||
@ -21,9 +21,9 @@ jobs:
|
||||
check-latest: true # optional
|
||||
# Used to specify whether caching is needed. Set to true, if you'd like to enable caching.
|
||||
cache: true # optional
|
||||
|
||||
|
||||
- name: Build
|
||||
run: go build -v ./...
|
||||
|
||||
|
||||
- name: Test
|
||||
run: go test -v ./... -json -coverprofile=coverage.out > test-report.out
|
||||
|
@ -34,7 +34,8 @@ To install the project, follow these steps:
|
||||
|
||||
## Setting Up Environment Variables
|
||||
|
||||
The program requires certain environment variables to be set.
|
||||
The program requires certain environment variables to be set.
|
||||
|
||||
```plaintext
|
||||
E621_API_KEY=
|
||||
E621_USERNAME=
|
||||
@ -93,7 +94,7 @@ The program supports the following databases.
|
||||
|
||||
### Neo4j
|
||||
|
||||
To improve performance, it is recommended to create indices on different nodes in Neo4j.
|
||||
To improve performance, it is recommended to create indices on different nodes in Neo4j.
|
||||
|
||||
Run the following commands to create the required indices:
|
||||
|
||||
@ -106,7 +107,7 @@ neo4j$ CREATE INDEX userIdIndex FOR (u:e621User) ON (u.e621ID);
|
||||
|
||||
### Memgraph:
|
||||
|
||||
To improve performance, it is recommended to create indices on different nodes in Memgraph.
|
||||
To improve performance, it is recommended to create indices on different nodes in Memgraph.
|
||||
|
||||
Run the following commands to create the required indices:
|
||||
|
||||
|
@ -2,11 +2,11 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/api"
|
||||
"e621_to_neo4j/database"
|
||||
neo4j "e621_to_neo4j/database/neo4j"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/utils"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/api"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/config"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/e621"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"net/http"
|
||||
"os"
|
||||
@ -30,11 +30,11 @@ func init() {
|
||||
|
||||
func main() {
|
||||
|
||||
var graphConnection database.GraphConnection
|
||||
var graphConnection logic.GraphConnection
|
||||
ctx := context.Background()
|
||||
|
||||
// Loads Config
|
||||
config, err := utils.LoadConfig()
|
||||
config, err := config.LoadConfig()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
@ -57,8 +57,8 @@ func main() {
|
||||
|
||||
log.Printf("Im ready!")
|
||||
|
||||
// Register the UserHandler with the "/user" route
|
||||
http.HandleFunc("/user", api.UserHandler(ctx, graphConnection, e621Client))
|
||||
// Register the ScapeUserFavourites with the "/user" route
|
||||
http.HandleFunc("/user", api.ScapeUserFavourites(ctx, graphConnection, e621Client))
|
||||
|
||||
// Start the HTTP server
|
||||
err = http.ListenAndServe(":8080", nil)
|
@ -1,14 +0,0 @@
|
||||
package e621
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// GetFavorites retrieves all favorites from the e621 API.
|
||||
func (c *Client) GetFavorites(_ context.Context, user models.E621User, page int64) func() (models.PostResponseWrapper, error) {
|
||||
URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page)
|
||||
e621Task := NewE621ApiTask[models.PostResponseWrapper](URIPath)
|
||||
return Schedule[models.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey)
|
||||
}
|
56
e621/task.go
56
e621/task.go
@ -1,56 +0,0 @@
|
||||
package e621
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/utils"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type e621APITask[T utils.DataType] struct {
|
||||
uri string
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) UriPath() string {
|
||||
return e.uri
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) HandleStatusCode(statusCode int) utils.DataResponse[T] {
|
||||
var err error
|
||||
switch statusCode {
|
||||
case 421:
|
||||
err = utils.RateLimitReachedError{}
|
||||
case 424:
|
||||
err = utils.InvalidParametersError{}
|
||||
case 520:
|
||||
err = utils.OriginConnectionTimeOutError{}
|
||||
case 522:
|
||||
err = utils.OriginConnectionTimeOutError{}
|
||||
case 524:
|
||||
err = utils.OriginConnectionTimeOutError{}
|
||||
case 525:
|
||||
err = utils.SSLHandshakeFailedError{}
|
||||
default:
|
||||
err = utils.StatusCodesToError(statusCode)
|
||||
}
|
||||
return utils.DataResponse[T]{Error: err}
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) HandleResponse(responseData *http.Response) utils.DataResponse[T] {
|
||||
var data T
|
||||
err := json.NewDecoder(responseData.Body).Decode(&data)
|
||||
defer responseData.Body.Close()
|
||||
if err != nil {
|
||||
return utils.DataResponse[T]{Error: err}
|
||||
}
|
||||
return utils.DataResponse[T]{Data: data}
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) HandleError(error error) utils.DataResponse[T] {
|
||||
return utils.DataResponse[T]{Error: error}
|
||||
}
|
||||
|
||||
func NewE621ApiTask[T utils.DataType](uri string) utils.Task[T] {
|
||||
return &e621APITask[T]{
|
||||
uri: uri,
|
||||
}
|
||||
}
|
@ -1,13 +0,0 @@
|
||||
package e621
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/e621/models"
|
||||
"fmt"
|
||||
)
|
||||
|
||||
// GetUserInfo retrieves the users information from e621 API.
|
||||
func (c *Client) GetUserInfo(username string) func() (models.E621User, error) {
|
||||
URIPath := fmt.Sprintf("users/%s.json", username)
|
||||
e621Task := NewE621ApiTask[models.E621User](URIPath)
|
||||
return Schedule[models.E621User](c.scheduler, e621Task, c.username, c.apiKey)
|
||||
}
|
2
go.mod
2
go.mod
@ -1,4 +1,4 @@
|
||||
module e621_to_neo4j
|
||||
module git.dragse.it/anthrove/e621-to-graph
|
||||
|
||||
go 1.20
|
||||
|
||||
|
@ -2,16 +2,16 @@ package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/database"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/services"
|
||||
"fmt"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/e621"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/service"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// UserHandler is the handler for the user API
|
||||
func UserHandler(ctx context.Context, graphConnection database.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
|
||||
// ScapeUserFavourites is the handler for the user API
|
||||
func ScapeUserFavourites(ctx context.Context, graphConnection logic.GraphConnection, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
log.Println("Request")
|
||||
if r.Method != http.MethodPost {
|
||||
@ -28,7 +28,7 @@ func UserHandler(ctx context.Context, graphConnection database.GraphConnection,
|
||||
}
|
||||
|
||||
// Perform further processing with the username
|
||||
go services.ScrapeUser(ctx, graphConnection, *e621Client, username)
|
||||
go service.ScrapeUser(ctx, graphConnection, *e621Client, username)
|
||||
|
||||
// Send a response
|
||||
w.WriteHeader(http.StatusOK)
|
@ -1,4 +1,4 @@
|
||||
package utils
|
||||
package config
|
||||
|
||||
import (
|
||||
"fmt"
|
@ -2,8 +2,8 @@ package neo4j
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/database"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j/config"
|
||||
)
|
||||
@ -12,7 +12,7 @@ type neo4jConnection struct {
|
||||
driver neo4j.DriverWithContext
|
||||
}
|
||||
|
||||
func NewNeo4JConnection() database.GraphConnection {
|
||||
func NewNeo4JConnection() logic.GraphConnection {
|
||||
return &neo4jConnection{}
|
||||
}
|
||||
|
||||
@ -44,7 +44,7 @@ func (c *neo4jConnection) UploadSource(ctx context.Context, SourceURL string) er
|
||||
return CreateSourceNode(ctx, c.driver, SourceURL)
|
||||
}
|
||||
|
||||
func (c *neo4jConnection) UploadUser(ctx context.Context, user models.E621User) error {
|
||||
func (c *neo4jConnection) UploadUser(ctx context.Context, user model.E621User) error {
|
||||
return CreateUserNode(ctx, c.driver, user)
|
||||
}
|
||||
func (c *neo4jConnection) Connect(ctx context.Context, endpoint string, username string, password string) error {
|
@ -1,4 +1,4 @@
|
||||
package models
|
||||
package model
|
||||
|
||||
type DBTag struct {
|
||||
Tag string
|
@ -2,7 +2,7 @@ package neo4j
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/database/neo4j/models"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/database/neo4j/model"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
)
|
||||
|
||||
@ -24,9 +24,9 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str
|
||||
return nil
|
||||
}
|
||||
|
||||
func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string) (models.DBTag, bool, error) {
|
||||
func GetTagNodeByName(ctx context.Context, driver neo4j.DriverWithContext, name string) (model.DBTag, bool, error) {
|
||||
|
||||
var tag models.DBTag
|
||||
var tag model.DBTag
|
||||
|
||||
query := `
|
||||
MATCH (u:e621Tag {e621Tag: $name})
|
||||
@ -48,7 +48,7 @@ func GetTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string
|
||||
e621Tag, _, _ := neo4j.GetRecordValue[string](record, "e621Tag")
|
||||
e621TagType, _, _ := neo4j.GetRecordValue[string](record, "e621TagType")
|
||||
|
||||
tag = models.DBTag{
|
||||
tag = model.DBTag{
|
||||
Tag: e621Tag,
|
||||
TagType: e621TagType,
|
||||
}
|
@ -2,11 +2,11 @@ package neo4j
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
)
|
||||
|
||||
func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error {
|
||||
func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user model.E621User) error {
|
||||
query := `
|
||||
MERGE (u:e621User {e621ID: $id, e621Username: $name})
|
||||
RETURN u
|
@ -1,21 +1,20 @@
|
||||
package e621
|
||||
|
||||
import "golang.org/x/time/rate"
|
||||
|
||||
const (
|
||||
baseURL = "https://e621.net"
|
||||
import (
|
||||
e621 "git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
// Client represents the e621 API client.
|
||||
type Client struct {
|
||||
apiKey string
|
||||
username string
|
||||
scheduler *Scheduler
|
||||
scheduler *e621.Scheduler
|
||||
}
|
||||
|
||||
// NewClient creates a new e621 API client.
|
||||
func NewClient(apiKey string, username string) *Client {
|
||||
scheduler := NewScheduler()
|
||||
scheduler := e621.NewScheduler()
|
||||
scheduler.SetLimiter(rate.NewLimiter(1, 2))
|
||||
return &Client{
|
||||
apiKey: apiKey,
|
15
internal/e621/favorites.go
Normal file
15
internal/e621/favorites.go
Normal file
@ -0,0 +1,15 @@
|
||||
package e621
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
|
||||
)
|
||||
|
||||
// GetFavorites retrieves all favorites from the e621 API.
|
||||
func (c *Client) GetFavorites(_ context.Context, user model.E621User, page int64) func() (model.PostResponseWrapper, error) {
|
||||
URIPath := fmt.Sprintf("favorites.json?user_id=%d&limit=%d&page=%d", user.ID, 320, page)
|
||||
e621Task := NewE621ApiTask[model.PostResponseWrapper](URIPath)
|
||||
return scheduler.Schedule[model.PostResponseWrapper](c.scheduler, e621Task, c.username, c.apiKey)
|
||||
}
|
@ -1,17 +1,21 @@
|
||||
package e621
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/utils"
|
||||
"fmt"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func ExecuteGetAPIRequest(schedulerTask utils.SchedulerTask) {
|
||||
const (
|
||||
BASEURL = "https://e621.net"
|
||||
)
|
||||
|
||||
func GetAPIRequest(schedulerTask queue.SchedulerTask) {
|
||||
var err error
|
||||
log.Debug("executing scheduler task")
|
||||
|
||||
url := fmt.Sprintf("%s/%s", baseURL, schedulerTask.UriPath())
|
||||
url := fmt.Sprintf("%s/%s", BASEURL, schedulerTask.UriPath())
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
schedulerTask.SendError(err)
|
@ -1,28 +1,29 @@
|
||||
package e621
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/utils"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"golang.org/x/time/rate"
|
||||
)
|
||||
|
||||
type Scheduler struct {
|
||||
queue utils.Queue
|
||||
queue queue.Queue
|
||||
limiter *rate.Limiter
|
||||
}
|
||||
|
||||
func NewScheduler() *Scheduler {
|
||||
scheduler := &Scheduler{
|
||||
queue: utils.NewQueue(),
|
||||
queue: queue.NewQueue(),
|
||||
limiter: nil,
|
||||
}
|
||||
go scheduler.StartExecutionHandler()
|
||||
return scheduler
|
||||
}
|
||||
|
||||
func Schedule[T utils.DataType](s *Scheduler, t utils.Task[T], username string, apiKey string) func() (T, error) {
|
||||
channel := make(chan utils.DataResponse[T])
|
||||
func Schedule[T e621.DataType](s *Scheduler, t e621.Task[T], username string, apiKey string) func() (T, error) {
|
||||
channel := make(chan e621.DataResponse[T])
|
||||
schedulerTask := NewSchedulerTaskImpl[T](t, channel, username, apiKey)
|
||||
log.Debug("Psuh task")
|
||||
err := s.queue.Push(schedulerTask)
|
||||
@ -55,7 +56,7 @@ func (s *Scheduler) StartExecutionHandler() {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ExecuteGetAPIRequest(task)
|
||||
GetAPIRequest(task)
|
||||
}
|
||||
|
||||
}
|
@ -1,13 +1,14 @@
|
||||
package e621
|
||||
package scheduler
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/utils"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/util/queue"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type schedulerTaskImpl[T utils.DataType] struct {
|
||||
task utils.Task[T]
|
||||
channel chan utils.DataResponse[T]
|
||||
type schedulerTaskImpl[T e621.DataType] struct {
|
||||
task e621.Task[T]
|
||||
channel chan e621.DataResponse[T]
|
||||
username string
|
||||
apiKey string
|
||||
}
|
||||
@ -16,7 +17,7 @@ func (s schedulerTaskImpl[T]) BasicAuth() (string, string) {
|
||||
return s.username, s.apiKey
|
||||
}
|
||||
|
||||
func NewSchedulerTaskImpl[T utils.DataType](task utils.Task[T], channel chan utils.DataResponse[T], username string, apiKey string) utils.SchedulerTask {
|
||||
func NewSchedulerTaskImpl[T e621.DataType](task e621.Task[T], channel chan e621.DataResponse[T], username string, apiKey string) queue.SchedulerTask {
|
||||
return &schedulerTaskImpl[T]{
|
||||
task: task,
|
||||
channel: channel,
|
57
internal/e621/task.go
Normal file
57
internal/e621/task.go
Normal file
@ -0,0 +1,57 @@
|
||||
package e621
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621"
|
||||
pgkError "git.dragse.it/anthrove/e621-to-graph/pkg/error"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type e621APITask[T e621.DataType] struct {
|
||||
uri string
|
||||
}
|
||||
|
||||
func NewE621ApiTask[T e621.DataType](uri string) e621.Task[T] {
|
||||
return &e621APITask[T]{
|
||||
uri: uri,
|
||||
}
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) UriPath() string {
|
||||
return e.uri
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) HandleStatusCode(statusCode int) e621.DataResponse[T] {
|
||||
var err error
|
||||
switch statusCode {
|
||||
case 421:
|
||||
err = pgkError.RateLimitReachedError{}
|
||||
case 424:
|
||||
err = pgkError.InvalidParametersError{}
|
||||
case 520:
|
||||
err = pgkError.OriginConnectionTimeOutError{}
|
||||
case 522:
|
||||
err = pgkError.OriginConnectionTimeOutError{}
|
||||
case 524:
|
||||
err = pgkError.OriginConnectionTimeOutError{}
|
||||
case 525:
|
||||
err = pgkError.SSLHandshakeFailedError{}
|
||||
default:
|
||||
err = pgkError.StatusCodesToError(statusCode)
|
||||
}
|
||||
return e621.DataResponse[T]{Error: err}
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) HandleResponse(responseData *http.Response) e621.DataResponse[T] {
|
||||
var data T
|
||||
err := json.NewDecoder(responseData.Body).Decode(&data)
|
||||
defer responseData.Body.Close()
|
||||
if err != nil {
|
||||
return e621.DataResponse[T]{Error: err}
|
||||
}
|
||||
return e621.DataResponse[T]{Data: data}
|
||||
}
|
||||
|
||||
func (e e621APITask[T]) HandleError(error error) e621.DataResponse[T] {
|
||||
return e621.DataResponse[T]{Error: error}
|
||||
}
|
14
internal/e621/user.go
Normal file
14
internal/e621/user.go
Normal file
@ -0,0 +1,14 @@
|
||||
package e621
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/e621/scheduler"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
|
||||
)
|
||||
|
||||
// GetUserInfo retrieves the users information from e621 API.
|
||||
func (c *Client) GetUserInfo(username string) func() (model.E621User, error) {
|
||||
URIPath := fmt.Sprintf("users/%s.json", username)
|
||||
e621Task := NewE621ApiTask[model.E621User](URIPath)
|
||||
return scheduler.Schedule[model.E621User](c.scheduler, e621Task, c.username, c.apiKey)
|
||||
}
|
@ -1,19 +1,19 @@
|
||||
package services
|
||||
package service
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/database"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"e621_to_neo4j/utils"
|
||||
"git.dragse.it/anthrove/e621-to-graph/internal/e621"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/util"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"time"
|
||||
)
|
||||
|
||||
func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e621Client e621.Client, username string) error {
|
||||
func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621Client e621.Client, username string) error {
|
||||
var err error
|
||||
var page int64 = 1
|
||||
var allFavorites []models.Post
|
||||
var allFavorites []model.Post
|
||||
|
||||
AwaitE621User := e621Client.GetUserInfo(username)
|
||||
e621User, err := AwaitE621User()
|
||||
@ -167,7 +167,7 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
|
||||
}
|
||||
|
||||
// uploadNodes uploads the post to the database and creates the nodes
|
||||
func uploadNodes(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
|
||||
func uploadNodes(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
|
||||
|
||||
uniqueGeneralTags := make([]string, 0)
|
||||
uniqueCharacterTags := make([]string, 0)
|
||||
@ -184,10 +184,10 @@ func uploadNodes(ctx context.Context, graphConnection database.GraphConnection,
|
||||
allCopyrightTags = append(allCopyrightTags, post.Tags.Copyright...)
|
||||
allArtistTags = append(allArtistTags, post.Tags.Artist...)
|
||||
|
||||
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
|
||||
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
|
||||
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
|
||||
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
|
||||
uniqueGeneralTags = util.UniqueNonEmptyElementsOf(allGeneralTags)
|
||||
uniqueCharacterTags = util.UniqueNonEmptyElementsOf(allCharacterTags)
|
||||
uniqueCopyrightTags = util.UniqueNonEmptyElementsOf(allCopyrightTags)
|
||||
uniqueArtistTags = util.UniqueNonEmptyElementsOf(allArtistTags)
|
||||
|
||||
err := graphConnection.UploadPost(ctx, post.ID)
|
||||
if err != nil {
|
||||
@ -233,7 +233,7 @@ func uploadNodes(ctx context.Context, graphConnection database.GraphConnection,
|
||||
}
|
||||
|
||||
// uploadPostToUserRelationship creates a relationship between the user and the post
|
||||
func uploadPostToUserRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post, e621User models.E621User) error {
|
||||
func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.E621User) error {
|
||||
err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -243,7 +243,7 @@ func uploadPostToUserRelationship(ctx context.Context, graphConnection database.
|
||||
}
|
||||
|
||||
// uploadSourceTagRelationship creates a relationship between the post and the source
|
||||
func uploadSourceTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
|
||||
func uploadSourceTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
|
||||
for _, source := range post.Sources {
|
||||
err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source)
|
||||
if err != nil {
|
||||
@ -256,7 +256,7 @@ func uploadSourceTagRelationship(ctx context.Context, graphConnection database.G
|
||||
}
|
||||
|
||||
// uploadGeneralTagRelationship creates a relationship between the post and the general tag
|
||||
func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
|
||||
func uploadGeneralTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
|
||||
for _, generalTag := range post.Tags.General {
|
||||
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag)
|
||||
if err != nil {
|
||||
@ -269,7 +269,7 @@ func uploadGeneralTagRelationship(ctx context.Context, graphConnection database.
|
||||
}
|
||||
|
||||
// uploadCharacterTagtRelationship creates a relationship between the post and the character tag
|
||||
func uploadCharacterTagtRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
|
||||
func uploadCharacterTagtRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
|
||||
for _, characterTag := range post.Tags.Character {
|
||||
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag)
|
||||
if err != nil {
|
||||
@ -282,7 +282,7 @@ func uploadCharacterTagtRelationship(ctx context.Context, graphConnection databa
|
||||
}
|
||||
|
||||
// uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag
|
||||
func uploadCopyrightTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
|
||||
func uploadCopyrightTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
|
||||
for _, copyrightTag := range post.Tags.Copyright {
|
||||
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag)
|
||||
if err != nil {
|
||||
@ -295,7 +295,7 @@ func uploadCopyrightTagRelationship(ctx context.Context, graphConnection databas
|
||||
}
|
||||
|
||||
// uploadArtistTagRelationship creates a relationship between the post and the artist tag
|
||||
func uploadArtistTagRelationship(ctx context.Context, graphConnection database.GraphConnection, post models.Post) error {
|
||||
func uploadArtistTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
|
||||
for _, artistTag := range post.Tags.Artist {
|
||||
err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag)
|
||||
if err != nil {
|
@ -1,4 +1,4 @@
|
||||
package models
|
||||
package model
|
||||
|
||||
import "encoding/json"
|
||||
|
@ -1,4 +1,4 @@
|
||||
package models
|
||||
package model
|
||||
|
||||
import "encoding/json"
|
||||
|
@ -1,7 +1,7 @@
|
||||
package utils
|
||||
package e621
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/e621/models"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
@ -11,7 +11,7 @@ type DataResponse[T DataType] struct {
|
||||
}
|
||||
|
||||
type DataType interface {
|
||||
models.E621User | models.PostResponseWrapper
|
||||
model.E621User | model.PostResponseWrapper
|
||||
}
|
||||
|
||||
type Task[T DataType] interface {
|
||||
@ -20,11 +20,3 @@ type Task[T DataType] interface {
|
||||
HandleStatusCode(statusCode int) DataResponse[T]
|
||||
HandleResponse(responseData *http.Response) DataResponse[T]
|
||||
}
|
||||
|
||||
type SchedulerTask interface {
|
||||
UriPath() string
|
||||
SendStatusCode(statusCode int)
|
||||
SendError(err error)
|
||||
SendResponse(response *http.Response)
|
||||
BasicAuth() (string, string)
|
||||
}
|
@ -1,4 +1,4 @@
|
||||
package utils
|
||||
package error
|
||||
|
||||
import "fmt"
|
||||
|
@ -1,22 +1,13 @@
|
||||
package database
|
||||
package logic
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
|
||||
)
|
||||
|
||||
// [X:Neo4J] Connection
|
||||
// [X:Neo4J] Upload User
|
||||
// [X:Neo4J] Upload Source
|
||||
// [X:Neo4J] Upload Post
|
||||
// [X:Neo4J] Upload Tag
|
||||
// [X:Neo4J] Relationship Post->Tag
|
||||
// [X:Neo4J] Relationship Post->Source
|
||||
// [X:Neo4J] Relationship User->Post
|
||||
|
||||
type GraphConnection interface {
|
||||
Connect(ctx context.Context, endpoint string, username string, password string) error
|
||||
UploadUser(ctx context.Context, user models.E621User) error
|
||||
UploadUser(ctx context.Context, user model.E621User) error
|
||||
UploadSource(ctx context.Context, SourceURL string) error
|
||||
UploadPost(ctx context.Context, e621ID int64) error
|
||||
UploadTag(ctx context.Context, name string, tagType string) error
|
@ -1,4 +1,4 @@
|
||||
package utils
|
||||
package util
|
||||
|
||||
// UniqueNonEmptyElementsOf returns a new slice containing unique non-empty elements from the input slice.
|
||||
// It removes duplicate elements and empty strings while preserving the order of appearance.
|
@ -1,4 +1,4 @@
|
||||
package utils
|
||||
package queue
|
||||
|
||||
import (
|
||||
"errors"
|
@ -1,4 +1,4 @@
|
||||
package utils
|
||||
package queue
|
||||
|
||||
import (
|
||||
"net/http"
|
13
pkg/util/queue/scheduler.go
Normal file
13
pkg/util/queue/scheduler.go
Normal file
@ -0,0 +1,13 @@
|
||||
package queue
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type SchedulerTask interface {
|
||||
UriPath() string
|
||||
SendStatusCode(statusCode int)
|
||||
SendError(err error)
|
||||
SendResponse(response *http.Response)
|
||||
BasicAuth() (string, string)
|
||||
}
|
Reference in New Issue
Block a user