e621-sdk-go/services/user.go

package services

import (
	"context"
	"e621_to_neo4j/e621"
	"e621_to_neo4j/e621/models"
	"e621_to_neo4j/neo4jAPI"
	"e621_to_neo4j/utils"
	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
	"log"
	"time"
)

func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error {
	var err error

	e621User, err := e621Client.GetUserInfo(username)
	if err != nil {
		return err
	}

	if e621User.IsBanned {
		log.Printf("User %s is banned from e621!", e621User.Name)
		return nil
	}

	log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID)

	err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("Getting favorites for user %s with id %d", e621User.Name, e621User.ID)
	start := time.Now()
	userFavorites, err := e621Client.GetFavorites(e621User)
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("User %s with id %d has %d favorites. Time took to scrape: %v", e621User.Name, e621User.ID, len(userFavorites), time.Since(start))

	startUploadPosts := time.Now()

	// Uploads all Tags, Posts as Nodes to Neo4j
	for i, post := range userFavorites {

		if exists, err := neo4jAPI.CheckUserToPostRelationship(ctx, driver, post.ID, e621User.ID); err == nil && exists {
			log.Printf("No new posts found for user %s with id %d", e621User.Name, e621User.ID)
			log.Printf("Last Post ID Found: %d", post.ID)
			break
		} else if err != nil {
			return err
		}

		start = time.Now()
		err = uploadNodes(ctx, driver, post)
		if err != nil {
			return err
		}
		log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))

		start := time.Now()
		err = uploadPostToUserRelationship(ctx, driver, post, e621User)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadSourceTagRelationship(ctx, driver, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadGeneralTagRelationship(ctx, driver, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadCharacterTagtRelationship(ctx, driver, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadCopyrightTagRelationship(ctx, driver, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadArtistTagRelationship(ctx, driver, post)
		if err != nil {
			log.Fatal(err)
			return err
		}
		log.Printf("Making relationship for user %s with id %d, %d for Post: %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))
	}
	log.Printf("Uploading all posts for user %s took: %v", username, time.Since(startUploadPosts))

	return nil
}

// uploadNodes uploads the post to the database and creates the nodes
func uploadNodes(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {

	uniqueGeneralTags := make([]string, 0)
	uniqueCharacterTags := make([]string, 0)
	uniqueCopyrightTags := make([]string, 0)
	uniqueArtistTags := make([]string, 0)

	allGeneralTags := make([]string, 0)
	allCharacterTags := make([]string, 0)
	allCopyrightTags := make([]string, 0)
	allArtistTags := make([]string, 0)

	allGeneralTags = append(allGeneralTags, post.Tags.General...)
	allCharacterTags = append(allCharacterTags, post.Tags.Character...)
	allCopyrightTags = append(allCopyrightTags, post.Tags.Character...)
	allArtistTags = append(allArtistTags, post.Tags.Artist...)

	uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
	uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
	uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
	uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)

	err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
	if err != nil {
		return err
	}

	// Uploads the source to the database
	for _, source := range post.Sources {
		err := neo4jAPI.CreateSourceNode(ctx, driver, source)
		if err != nil {
			return err
		}
	}

	for _, uniqueGeneralTag := range uniqueGeneralTags {
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
		if err != nil {
			return err
		}
	}
	for _, uniqueCharacterTag := range uniqueCharacterTags {
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
		if err != nil {
			return err
		}
	}

	for _, uniqueCopyrightTag := range uniqueCopyrightTags {
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
		if err != nil {
			return err
		}
	}

	for _, uniqueArtistTag := range uniqueArtistTags {
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
		if err != nil {
			return err
		}
	}

	return nil
}

// uploadPostToUserRelationship creates a relationship between the user and the post
func uploadPostToUserRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post, e621User models.E621User) error {
	err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
	if err != nil {
		return err
	}
	// log.Printf("Created UserToPostRelationship for user: %s to post: %d", e621User.Name, post.ID)
	return nil
}

// uploadSourceTagRelationship creates a relationship between the post and the source
func uploadSourceTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
	for _, source := range post.Sources {
		err := neo4jAPI.PostToSourceRelationship(ctx, driver, post.ID, source)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToSourceRelationship for Post: %d to source: %s", post.ID, source)

	}
	return nil
}

// uploadGeneralTagRelationship creates a relationship between the post and the general tag
func uploadGeneralTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
	for _, generalTag := range post.Tags.General {
		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, generalTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to general tag: %s", post.ID, generalTag)

	}
	return nil
}

// uploadCharacterTagtRelationship creates a relationship between the post and the character tag
func uploadCharacterTagtRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
	for _, characterTag := range post.Tags.Character {
		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, characterTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to character tag: %s", post.ID, characterTag)

	}
	return nil
}

// uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag
func uploadCopyrightTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
	for _, copyrightTag := range post.Tags.Copyright {
		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, copyrightTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to copyrigh tag: %s", post.ID, copyrightTag)

	}
	return nil
}

// uploadArtistTagRelationship creates a relationship between the post and the artist tag
func uploadArtistTagRelationship(ctx context.Context, driver neo4j.DriverWithContext, post models.Post) error {
	for _, artistTag := range post.Tags.Artist {
		err := neo4jAPI.PostToTagRelationship(ctx, driver, post.ID, artistTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to artist tag: %s", post.ID, artistTag)

	}
	return nil
}

//11min für Selloo, simultan mit mutt_jake
//1h58m53 für mutt_jake, mit Selloo