anthrove/e621-sdk-go
Archived
4
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases Wiki Activity

added user api implementation

Browse Source
This commit is contained in:
David Janowski 2023-05-24 16:05:27 +02:00
parent dc4cf3be42
commit 10c0247f3c
11 changed files with 247 additions and 222 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
api/user.go Normal file
View File

@ -0,0 +1,36 @@
package api
import (
"context"
"e621_to_neo4j/e621"
"e621_to_neo4j/services"
"fmt"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"net/http"
)
// UserHandler is the handler for the user API
func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
return func(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodPost {
w.WriteHeader(http.StatusMethodNotAllowed)
fmt.Fprintf(w, "Only POST requests are allowed")
return
}
username := r.FormValue("username")
if username == "" {
w.WriteHeader(http.StatusBadRequest)
fmt.Fprintf(w, "Username is required")
return
}
// Perform further processing with the username
services.ScrapeUser(ctx, driver, *e621Client, username)
// Send a response
w.WriteHeader(http.StatusOK)
fmt.Fprintf(w, "Username %s processed successfully", username)
}
}

7
docker-compose.yml
View File

@ -5,4 +5,9 @@ services:
ports:
- '7474:7474'
- '7473:7473'
- '7687:7687'
- '7687:7687'
volumes:
- 'neo4j_data:/bitnami'
volumes:
neo4j_data:
driver: local

2
e621/api/client.go → e621/client.go
View File

@ -1,4 +1,4 @@
package api
package e621
import (
"net/http"

4
e621/api/favorite.go → e621/favorite.go
View File

@ -1,7 +1,7 @@
package api
package e621
import (
"e621_to_neo4j/e621/api/models"
"e621_to_neo4j/e621/models"
"fmt"
"io"
"log"

8
e621/api/models/post.go → e621/models/post.go
View File

@ -2,17 +2,17 @@ package models
import "encoding/json"
func UnmarshalE621Post(data []byte) (E621Post, error) {
var r E621Post
func UnmarshalE621Post(data []byte) (APIe621Post, error) {
var r APIe621Post
err := json.Unmarshal(data, &r)
return r, err
}
func (r *E621Post) Marshal() ([]byte, error) {
func (r *APIe621Post) Marshal() ([]byte, error) {
return json.Marshal(r)
}
type E621Post struct {
type APIe621Post struct {
Posts []Post `json:"posts"`
}

0
e621/api/models/user.go → e621/models/user.go
View File

4
e621/api/users.go → e621/users.go
View File

@ -1,7 +1,7 @@
package api
package e621
import (
"e621_to_neo4j/e621/api/models"
"e621_to_neo4j/e621/models"
"fmt"
"io"
"net/http"

228
main.go
View File

@ -2,21 +2,26 @@ package main
import (
"context"
"e621_to_neo4j/e621/api"
"e621_to_neo4j/e621/api/models"
"e621_to_neo4j/api"
"e621_to_neo4j/e621"
"e621_to_neo4j/neo4jAPI"
"e621_to_neo4j/utils"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"log"
"time"
"net/http"
)
const USER = "selloo"
func main() {
// Loads Config
config, err := utils.LoadConfig()
if err != nil {
log.Println(err)
}
// Connect to Neo4j DB
driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
if err != nil {
log.Fatal(err)
@ -30,215 +35,16 @@ func main() {
}
}(driver, ctx)
e621Client := api.NewClient(config.E621APIKey, config.E621Username)
user, err := e621Client.GetUserInfo("selloo")
// Initialize the e621API
e621Client := e621.NewClient(config.E621APIKey, config.E621Username)
// Register the UserHandler with the "/user" route
http.HandleFunc("/user", api.UserHandler(ctx, driver, e621Client))
// Start the HTTP server
err = http.ListenAndServe(":8080", nil)
if err != nil {
log.Fatal(err)
return
}
err = neo4jAPI.CreateUserNode(ctx, driver, user)
if err != nil {
log.Fatal(err)
}
favorites, err := e621Client.GetFavorites(user.Name)
if err != nil {
log.Fatal(err)
}
start := time.Now()
//uploadTags(favorites, ctx, driver)
elapsed := time.Since(start)
log.Printf("upload of Tags took %d", elapsed)
start = time.Now()
//uploadPosts(favorites, ctx, driver)
elapsed = time.Since(start)
log.Printf("upload of Posts took %d", elapsed)
start = time.Now()
//uploadSources(favorites, ctx, driver)
elapsed = time.Since(start)
log.Printf("upload of Sources took %d", elapsed)
start = time.Now()
// err = makePostToTagRelationship(favorites, ctx, driver)
elapsed = time.Since(start)
log.Printf("upload of Sources took %v", elapsed)
start = time.Now()
// makeUserToPostRelationship(favorites, user, ctx, driver)
elapsed = time.Since(start)
log.Printf("upload of Sources took %v", elapsed)
start = time.Now()
err = makePostToSourceRelationship(favorites, ctx, driver)
if err != nil {
log.Fatal(err)
}
elapsed = time.Since(start)
log.Printf("upload of Sources took %v", elapsed)
}
func uploadPosts(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
for _, fav := range posts {
log.Printf("e621 PostID: %d", fav.ID)
err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID)
if err != nil {
return err
}
}
return nil
}
func uploadTags(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
uniqueGeneralTags := make([]string, 0)
uniqueCharacterTags := make([]string, 0)
uniqueCopyrightTags := make([]string, 0)
uniqueArtistTags := make([]string, 0)
allGeneralTags := make([]string, 0)
allCharacterTags := make([]string, 0)
allCopyrightTags := make([]string, 0)
allArtistTags := make([]string, 0)
// add all tags together
for _, fav := range posts {
allGeneralTags = append(allGeneralTags, fav.Tags.General...)
allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
allArtistTags = append(allArtistTags, fav.Tags.Artist...)
}
// Process General PostTags
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
// Process Character PostTags
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
// Process Copyright PostTags
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
// Process Artist PostTags
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
for _, uniqueGeneralTag := range uniqueGeneralTags {
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
if err != nil {
return err
}
}
for _, uniqueCharacterTag := range uniqueCharacterTags {
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
if err != nil {
return err
}
}
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
if err != nil {
return err
}
}
for _, uniqueArtistTag := range uniqueArtistTags {
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
if err != nil {
return err
}
}
return nil
}
func uploadSources(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
for _, fav := range posts {
for _, source := range fav.Sources {
log.Printf("e621 Source: %s", source)
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
if err != nil {
return err
}
}
}
return nil
}
func makePostToTagRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
for _, fav := range posts {
id := fav.ID
for _, generalTags := range fav.Tags.General {
log.Printf("PostID: %d has General Tag: %s", id, generalTags)
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTags)
if err != nil {
return err
}
}
for _, characterTags := range fav.Tags.Character {
log.Printf("PostID: %d has Character Tag: %s", id, characterTags)
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTags)
if err != nil {
return err
}
}
for _, copyrightTags := range fav.Tags.Copyright {
log.Printf("PostID: %d has Copyright Tag: %s", id, copyrightTags)
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTags)
if err != nil {
return err
}
}
for _, artistTags := range fav.Tags.Artist {
log.Printf("PostID: %d has Artist Tag: %s", id, artistTags)
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTags)
if err != nil {
return err
}
}
}
return nil
}
func makeUserToPostRelationship(posts []models.Post, user models.E621User, ctx context.Context, driver neo4j.DriverWithContext) error {
for _, post := range posts {
log.Printf("UserID: %d has PostID: %d in his favorites", user.ID, post.ID)
err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, user.ID)
if err != nil {
return err
}
}
return nil
}
func makePostToSourceRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
for _, post := range posts {
postID := post.ID
for _, source := range post.Sources {
log.Printf("PostID: %d has Source URL: %s", postID, source)
err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
if err != nil {
return err
}
}
}
return nil
}

25
neo4jAPI/tag.go
View File

@ -24,6 +24,31 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str
return nil
}
func CreateTagNode2(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error {
session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
defer session.Close(ctx)
query := `
MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType})
RETURN u
`
params := map[string]interface{}{
"name": name,
"tagType": tagType,
}
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (any, error) {
result, err := tx.Run(ctx, query, params)
if err != nil {
return nil, err
}
return result.Consume(ctx)
})
return err
}
func GetAllTagNodes(ctx context.Context, driver neo4j.DriverWithContext, tagType string) ([]string, error) {
query := `
MATCH (u:e621Tag {e621TagType: $tagType})

2
neo4jAPI/user.go
View File

@ -2,7 +2,7 @@ package neo4jAPI
import (
"context"
"e621_to_neo4j/e621/api/models"
"e621_to_neo4j/e621/models"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"log"
)

153
services/user.go Normal file
View File

@ -0,0 +1,153 @@
package services
import (
"context"
"e621_to_neo4j/e621"
"e621_to_neo4j/neo4jAPI"
"e621_to_neo4j/utils"
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
"log"
"time"
)
func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error {
var err error
e621User, err := e621Client.GetUserInfo(username)
if err != nil {
return err
}
err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
if err != nil {
log.Fatal(err)
}
userFavorites, err := e621Client.GetFavorites(e621User.Name)
if err != nil {
log.Fatal(err)
}
startUploadPosts := time.Now()
for i, post := range userFavorites {
start := time.Now()
uniqueGeneralTags := make([]string, 0)
uniqueCharacterTags := make([]string, 0)
uniqueCopyrightTags := make([]string, 0)
uniqueArtistTags := make([]string, 0)
allGeneralTags := make([]string, 0)
allCharacterTags := make([]string, 0)
allCopyrightTags := make([]string, 0)
allArtistTags := make([]string, 0)
allGeneralTags = append(allGeneralTags, post.Tags.General...)
allCharacterTags = append(allCharacterTags, post.Tags.Character...)
allCopyrightTags = append(allCopyrightTags, post.Tags.Character...)
allArtistTags = append(allArtistTags, post.Tags.Artist...)
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
// Uploads post to database
err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
if err != nil {
return err
}
// Uploads the source to the database
for _, source := range post.Sources {
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
if err != nil {
return err
}
}
for _, uniqueGeneralTag := range uniqueGeneralTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
if err != nil {
return err
}
}
for _, uniqueCharacterTag := range uniqueCharacterTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
if err != nil {
return err
}
}
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
if err != nil {
return err
}
}
for _, uniqueArtistTag := range uniqueArtistTags {
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
if err != nil {
return err
}
}
log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start))
}
for _, post := range userFavorites {
id := post.ID
err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
if err != nil {
return err
}
log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID)
postID := post.ID
for _, source := range post.Sources {
err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
if err != nil {
return err
}
log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source)
}
for _, generalTag := range post.Tags.General {
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag)
if err != nil {
log.Println(err)
}
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag)
}
for _, characterTag := range post.Tags.Character {
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag)
if err != nil {
log.Println(err)
}
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag)
}
for _, copyrightTag := range post.Tags.Copyright {
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag)
if err != nil {
log.Println(err)
}
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag)
}
for _, artistTag := range post.Tags.Artist {
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag)
if err != nil {
log.Println(err)
}
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag)
}
}
log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts))
return nil
}