e621-sdk-go/internal/service/manager.go

package service

import (
	"context"
	"git.dragse.it/anthrove/e621-to-graph/internal/e621"
	"git.dragse.it/anthrove/e621-to-graph/pkg/e621/model"
	"git.dragse.it/anthrove/e621-to-graph/pkg/logic"
	"git.dragse.it/anthrove/e621-to-graph/pkg/util"
	log "github.com/sirupsen/logrus"
	"time"
)

func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621Client e621.Client, username string) error {
	var err error
	var page int64 = 1
	var allFavorites []model.Post

	AwaitE621User := e621Client.GetUserInfo(username)
	e621User, err := AwaitE621User()
	if err != nil {
		log.Info(err)
		return err
	}

	if e621User.IsBanned {
		log.WithFields(log.Fields{
			"e621_username":  e621User.Name,
			"e621_user_id":   e621User.ID,
			"e621_user_bann": e621User.IsBanned,
		}).Info("service: user is Banned")
		return nil
	}

	log.WithFields(log.Fields{
		"e621_username": e621User.Name,
		"e621_user_id":  e621User.ID,
	}).Info("service: processing user")

	err = graphConnection.UploadUser(ctx, e621User)
	if err != nil {
		log.Fatal(err)
	}

	log.WithFields(log.Fields{
		"e621_username": e621User.Name,
		"e621_user_id":  e621User.ID,
	}).Info("service: start processing favorites")
	start := time.Now()

	for {
		AwaitE621Favorites := e621Client.GetFavorites(ctx, e621User, page)

		favoritesPage, err := AwaitE621Favorites()
		if err != nil {
			log.Error(err.Error())
		}

		// Append the fetched posts to the result slice
		allFavorites = append(allFavorites, favoritesPage.Posts...)

		// If no more posts are returned, return the accumulated favorites
		if len(favoritesPage.Posts) == 0 {
			break
		}

		// Update the last post ID for the next page request
		page += 1

	}

	// Uploads all Tags, Posts as Nodes to Neo4j
	for i, post := range allFavorites {
		if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists {
			log.WithFields(log.Fields{
				"e621_username": e621User.Name,
				"e621_user_id":  e621User.ID,
				"last_post_id":  post.ID,
			}).Info("service: no new favorites found")
			break
		} else if err != nil {
			return err
		}

		start = time.Now()
		err = uploadNodes(ctx, graphConnection, post)
		if err != nil {
			return err
		}
		log.WithFields(log.Fields{
			"e621_username": e621User.Name,
			"e621_user_id":  e621User.ID,
			"post_number":   i,
			"post_amount":   len(allFavorites),
			"post_id":       post.ID,
			"upload_time":   time.Since(start),
		}).Debug("service: uploading post")

		start := time.Now()
		err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadSourceTagRelationship(ctx, graphConnection, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadGeneralTagRelationship(ctx, graphConnection, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadCharacterTagtRelationship(ctx, graphConnection, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadCopyrightTagRelationship(ctx, graphConnection, post)
		if err != nil {
			log.Fatal(err)
			return err
		}

		err = uploadArtistTagRelationship(ctx, graphConnection, post)
		if err != nil {
			log.Fatal(err)
			return err
		}
		log.WithFields(log.Fields{
			"e621_username": e621User.Name,
			"e621_user_id":  e621User.ID,
			"post_number":   i,
			"post_amount":   len(allFavorites),
			"post_id":       post.ID,
			"upload_time":   time.Since(start),
		}).Debug("service: making relationship")
	}
	log.WithFields(log.Fields{
		"e621_username": e621User.Name,
		"e621_user_id":  e621User.ID,
		"post_amount":   len(allFavorites),
		"scrape_time":   time.Since(start),
	}).Info("service: finished processing favorites")

	return nil
}

// uploadNodes uploads the post to the database and creates the nodes
func uploadNodes(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {

	uniqueGeneralTags := make([]string, 0)
	uniqueCharacterTags := make([]string, 0)
	uniqueCopyrightTags := make([]string, 0)
	uniqueArtistTags := make([]string, 0)

	allGeneralTags := make([]string, 0)
	allCharacterTags := make([]string, 0)
	allCopyrightTags := make([]string, 0)
	allArtistTags := make([]string, 0)

	allGeneralTags = append(allGeneralTags, post.Tags.General...)
	allCharacterTags = append(allCharacterTags, post.Tags.Character...)
	allCopyrightTags = append(allCopyrightTags, post.Tags.Copyright...)
	allArtistTags = append(allArtistTags, post.Tags.Artist...)

	uniqueGeneralTags = util.UniqueNonEmptyElementsOf(allGeneralTags)
	uniqueCharacterTags = util.UniqueNonEmptyElementsOf(allCharacterTags)
	uniqueCopyrightTags = util.UniqueNonEmptyElementsOf(allCopyrightTags)
	uniqueArtistTags = util.UniqueNonEmptyElementsOf(allArtistTags)

	err := graphConnection.UploadPost(ctx, post.ID)
	if err != nil {
		return err
	}

	// Uploads the source to the database
	for _, source := range post.Sources {
		err := graphConnection.UploadSource(ctx, source)
		if err != nil {
			return err
		}
	}

	for _, uniqueGeneralTag := range uniqueGeneralTags {
		err := graphConnection.UploadTag(ctx, uniqueGeneralTag, "general")
		if err != nil {
			return err
		}
	}
	for _, uniqueCharacterTag := range uniqueCharacterTags {
		err := graphConnection.UploadTag(ctx, uniqueCharacterTag, "character")
		if err != nil {
			return err
		}
	}

	for _, uniqueCopyrightTag := range uniqueCopyrightTags {
		err := graphConnection.UploadTag(ctx, uniqueCopyrightTag, "copyright")
		if err != nil {
			return err
		}
	}

	for _, uniqueArtistTag := range uniqueArtistTags {
		err := graphConnection.UploadTag(ctx, uniqueArtistTag, "artist")
		if err != nil {
			return err
		}
	}

	return nil
}

// uploadPostToUserRelationship creates a relationship between the user and the post
func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.E621User) error {
	err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID)
	if err != nil {
		return err
	}
	// log.Printf("Created UserToPostRelationship for user: %s to post: %d", e621User.Name, post.ID)
	return nil
}

// uploadSourceTagRelationship creates a relationship between the post and the source
func uploadSourceTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
	for _, source := range post.Sources {
		err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToSourceRelationship for Post: %d to source: %s", post.ID, source)

	}
	return nil
}

// uploadGeneralTagRelationship creates a relationship between the post and the general tag
func uploadGeneralTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
	for _, generalTag := range post.Tags.General {
		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to general tag: %s", post.ID, generalTag)

	}
	return nil
}

// uploadCharacterTagtRelationship creates a relationship between the post and the character tag
func uploadCharacterTagtRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
	for _, characterTag := range post.Tags.Character {
		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to character tag: %s", post.ID, characterTag)

	}
	return nil
}

// uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag
func uploadCopyrightTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
	for _, copyrightTag := range post.Tags.Copyright {
		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to copyright tag: %s", post.ID, copyrightTag)

	}
	return nil
}

// uploadArtistTagRelationship creates a relationship between the post and the artist tag
func uploadArtistTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error {
	for _, artistTag := range post.Tags.Artist {
		err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag)
		if err != nil {
			return err
		}
		// log.Printf("Created PostToTagRelationship for post: %d to artist tag: %s", post.ID, artistTag)

	}
	return nil
}