added user api implementation
This commit is contained in:
parent
dc4cf3be42
commit
10c0247f3c
36
api/user.go
Normal file
36
api/user.go
Normal file
@ -0,0 +1,36 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/services"
|
||||
"fmt"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
// UserHandler is the handler for the user API
|
||||
func UserHandler(ctx context.Context, driver neo4j.DriverWithContext, e621Client *e621.Client) func(response http.ResponseWriter, request *http.Request) {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodPost {
|
||||
w.WriteHeader(http.StatusMethodNotAllowed)
|
||||
fmt.Fprintf(w, "Only POST requests are allowed")
|
||||
return
|
||||
}
|
||||
|
||||
username := r.FormValue("username")
|
||||
if username == "" {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
fmt.Fprintf(w, "Username is required")
|
||||
return
|
||||
}
|
||||
|
||||
// Perform further processing with the username
|
||||
services.ScrapeUser(ctx, driver, *e621Client, username)
|
||||
|
||||
// Send a response
|
||||
w.WriteHeader(http.StatusOK)
|
||||
fmt.Fprintf(w, "Username %s processed successfully", username)
|
||||
|
||||
}
|
||||
}
|
@ -5,4 +5,9 @@ services:
|
||||
ports:
|
||||
- '7474:7474'
|
||||
- '7473:7473'
|
||||
- '7687:7687'
|
||||
- '7687:7687'
|
||||
volumes:
|
||||
- 'neo4j_data:/bitnami'
|
||||
volumes:
|
||||
neo4j_data:
|
||||
driver: local
|
@ -1,4 +1,4 @@
|
||||
package api
|
||||
package e621
|
||||
|
||||
import (
|
||||
"net/http"
|
@ -1,7 +1,7 @@
|
||||
package api
|
||||
package e621
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
@ -2,17 +2,17 @@ package models
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
func UnmarshalE621Post(data []byte) (E621Post, error) {
|
||||
var r E621Post
|
||||
func UnmarshalE621Post(data []byte) (APIe621Post, error) {
|
||||
var r APIe621Post
|
||||
err := json.Unmarshal(data, &r)
|
||||
return r, err
|
||||
}
|
||||
|
||||
func (r *E621Post) Marshal() ([]byte, error) {
|
||||
func (r *APIe621Post) Marshal() ([]byte, error) {
|
||||
return json.Marshal(r)
|
||||
}
|
||||
|
||||
type E621Post struct {
|
||||
type APIe621Post struct {
|
||||
Posts []Post `json:"posts"`
|
||||
}
|
||||
|
@ -1,7 +1,7 @@
|
||||
package api
|
||||
package e621
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
228
main.go
228
main.go
@ -2,21 +2,26 @@ package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/api"
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"e621_to_neo4j/api"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/neo4jAPI"
|
||||
"e621_to_neo4j/utils"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"log"
|
||||
"time"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
const USER = "selloo"
|
||||
|
||||
func main() {
|
||||
|
||||
// Loads Config
|
||||
config, err := utils.LoadConfig()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
// Connect to Neo4j DB
|
||||
driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
@ -30,215 +35,16 @@ func main() {
|
||||
}
|
||||
}(driver, ctx)
|
||||
|
||||
e621Client := api.NewClient(config.E621APIKey, config.E621Username)
|
||||
user, err := e621Client.GetUserInfo("selloo")
|
||||
// Initialize the e621API
|
||||
e621Client := e621.NewClient(config.E621APIKey, config.E621Username)
|
||||
|
||||
// Register the UserHandler with the "/user" route
|
||||
http.HandleFunc("/user", api.UserHandler(ctx, driver, e621Client))
|
||||
|
||||
// Start the HTTP server
|
||||
err = http.ListenAndServe(":8080", nil)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return
|
||||
}
|
||||
|
||||
err = neo4jAPI.CreateUserNode(ctx, driver, user)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
favorites, err := e621Client.GetFavorites(user.Name)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
//uploadTags(favorites, ctx, driver)
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("upload of Tags took %d", elapsed)
|
||||
|
||||
start = time.Now()
|
||||
//uploadPosts(favorites, ctx, driver)
|
||||
elapsed = time.Since(start)
|
||||
log.Printf("upload of Posts took %d", elapsed)
|
||||
|
||||
start = time.Now()
|
||||
//uploadSources(favorites, ctx, driver)
|
||||
elapsed = time.Since(start)
|
||||
log.Printf("upload of Sources took %d", elapsed)
|
||||
|
||||
start = time.Now()
|
||||
// err = makePostToTagRelationship(favorites, ctx, driver)
|
||||
elapsed = time.Since(start)
|
||||
log.Printf("upload of Sources took %v", elapsed)
|
||||
|
||||
start = time.Now()
|
||||
// makeUserToPostRelationship(favorites, user, ctx, driver)
|
||||
elapsed = time.Since(start)
|
||||
log.Printf("upload of Sources took %v", elapsed)
|
||||
|
||||
start = time.Now()
|
||||
err = makePostToSourceRelationship(favorites, ctx, driver)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
elapsed = time.Since(start)
|
||||
log.Printf("upload of Sources took %v", elapsed)
|
||||
|
||||
}
|
||||
func uploadPosts(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
for _, fav := range posts {
|
||||
log.Printf("e621 PostID: %d", fav.ID)
|
||||
err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
func uploadTags(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
uniqueGeneralTags := make([]string, 0)
|
||||
uniqueCharacterTags := make([]string, 0)
|
||||
uniqueCopyrightTags := make([]string, 0)
|
||||
uniqueArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags := make([]string, 0)
|
||||
allCharacterTags := make([]string, 0)
|
||||
allCopyrightTags := make([]string, 0)
|
||||
allArtistTags := make([]string, 0)
|
||||
|
||||
// add all tags together
|
||||
for _, fav := range posts {
|
||||
allGeneralTags = append(allGeneralTags, fav.Tags.General...)
|
||||
allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
|
||||
allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
|
||||
allArtistTags = append(allArtistTags, fav.Tags.Artist...)
|
||||
}
|
||||
|
||||
// Process General PostTags
|
||||
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
|
||||
|
||||
// Process Character PostTags
|
||||
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
|
||||
|
||||
// Process Copyright PostTags
|
||||
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
|
||||
|
||||
// Process Artist PostTags
|
||||
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
|
||||
|
||||
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
|
||||
|
||||
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
|
||||
|
||||
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
|
||||
|
||||
for _, uniqueGeneralTag := range uniqueGeneralTags {
|
||||
|
||||
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, uniqueCharacterTag := range uniqueCharacterTags {
|
||||
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
|
||||
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueArtistTag := range uniqueArtistTags {
|
||||
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func uploadSources(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
for _, fav := range posts {
|
||||
for _, source := range fav.Sources {
|
||||
log.Printf("e621 Source: %s", source)
|
||||
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makePostToTagRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
|
||||
for _, fav := range posts {
|
||||
id := fav.ID
|
||||
for _, generalTags := range fav.Tags.General {
|
||||
log.Printf("PostID: %d has General Tag: %s", id, generalTags)
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, characterTags := range fav.Tags.Character {
|
||||
log.Printf("PostID: %d has Character Tag: %s", id, characterTags)
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, copyrightTags := range fav.Tags.Copyright {
|
||||
log.Printf("PostID: %d has Copyright Tag: %s", id, copyrightTags)
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, artistTags := range fav.Tags.Artist {
|
||||
log.Printf("PostID: %d has Artist Tag: %s", id, artistTags)
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTags)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func makeUserToPostRelationship(posts []models.Post, user models.E621User, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
|
||||
for _, post := range posts {
|
||||
log.Printf("UserID: %d has PostID: %d in his favorites", user.ID, post.ID)
|
||||
err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, user.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func makePostToSourceRelationship(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
|
||||
for _, post := range posts {
|
||||
postID := post.ID
|
||||
for _, source := range post.Sources {
|
||||
log.Printf("PostID: %d has Source URL: %s", postID, source)
|
||||
err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -24,6 +24,31 @@ func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name str
|
||||
return nil
|
||||
}
|
||||
|
||||
func CreateTagNode2(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error {
|
||||
session := driver.NewSession(ctx, neo4j.SessionConfig{AccessMode: neo4j.AccessModeWrite})
|
||||
defer session.Close(ctx)
|
||||
|
||||
query := `
|
||||
MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType})
|
||||
RETURN u
|
||||
`
|
||||
params := map[string]interface{}{
|
||||
"name": name,
|
||||
"tagType": tagType,
|
||||
}
|
||||
|
||||
_, err := session.ExecuteWrite(ctx, func(tx neo4j.ManagedTransaction) (any, error) {
|
||||
result, err := tx.Run(ctx, query, params)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return result.Consume(ctx)
|
||||
})
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func GetAllTagNodes(ctx context.Context, driver neo4j.DriverWithContext, tagType string) ([]string, error) {
|
||||
query := `
|
||||
MATCH (u:e621Tag {e621TagType: $tagType})
|
||||
|
@ -2,7 +2,7 @@ package neo4jAPI
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"log"
|
||||
)
|
||||
|
153
services/user.go
Normal file
153
services/user.go
Normal file
@ -0,0 +1,153 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/neo4jAPI"
|
||||
"e621_to_neo4j/utils"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error {
|
||||
var err error
|
||||
|
||||
e621User, err := e621Client.GetUserInfo(username)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
userFavorites, err := e621Client.GetFavorites(e621User.Name)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
startUploadPosts := time.Now()
|
||||
for i, post := range userFavorites {
|
||||
start := time.Now()
|
||||
|
||||
uniqueGeneralTags := make([]string, 0)
|
||||
uniqueCharacterTags := make([]string, 0)
|
||||
uniqueCopyrightTags := make([]string, 0)
|
||||
uniqueArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags := make([]string, 0)
|
||||
allCharacterTags := make([]string, 0)
|
||||
allCopyrightTags := make([]string, 0)
|
||||
allArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags = append(allGeneralTags, post.Tags.General...)
|
||||
allCharacterTags = append(allCharacterTags, post.Tags.Character...)
|
||||
allCopyrightTags = append(allCopyrightTags, post.Tags.Character...)
|
||||
allArtistTags = append(allArtistTags, post.Tags.Artist...)
|
||||
|
||||
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
|
||||
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
|
||||
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
|
||||
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
|
||||
|
||||
// Uploads post to database
|
||||
err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Uploads the source to the database
|
||||
for _, source := range post.Sources {
|
||||
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueGeneralTag := range uniqueGeneralTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, uniqueCharacterTag := range uniqueCharacterTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueArtistTag := range uniqueArtistTags {
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start))
|
||||
}
|
||||
|
||||
for _, post := range userFavorites {
|
||||
id := post.ID
|
||||
|
||||
err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID)
|
||||
|
||||
postID := post.ID
|
||||
for _, source := range post.Sources {
|
||||
err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source)
|
||||
|
||||
}
|
||||
|
||||
for _, generalTag := range post.Tags.General {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag)
|
||||
|
||||
}
|
||||
for _, characterTag := range post.Tags.Character {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag)
|
||||
|
||||
}
|
||||
for _, copyrightTag := range post.Tags.Copyright {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag)
|
||||
|
||||
}
|
||||
for _, artistTag := range post.Tags.Artist {
|
||||
err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag)
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts))
|
||||
return nil
|
||||
}
|
Reference in New Issue
Block a user