e621-sdk-go/services/user.go

package services

import (
	"context"
	"e621_to_neo4j/e621"
	"e621_to_neo4j/neo4jAPI"
	"e621_to_neo4j/utils"
	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
	"log"
	"time"
)

func ScrapeUser(ctx context.Context, driver neo4j.DriverWithContext, e621Client e621.Client, username string) error {
	var err error

	e621User, err := e621Client.GetUserInfo(username)
	if err != nil {
		return err
	}

	err = neo4jAPI.CreateUserNode(ctx, driver, e621User)
	if err != nil {
		log.Fatal(err)
	}

	userFavorites, err := e621Client.GetFavorites(e621User.Name)
	if err != nil {
		log.Fatal(err)
	}

	startUploadPosts := time.Now()
	for i, post := range userFavorites {
		start := time.Now()

		uniqueGeneralTags := make([]string, 0)
		uniqueCharacterTags := make([]string, 0)
		uniqueCopyrightTags := make([]string, 0)
		uniqueArtistTags := make([]string, 0)

		allGeneralTags := make([]string, 0)
		allCharacterTags := make([]string, 0)
		allCopyrightTags := make([]string, 0)
		allArtistTags := make([]string, 0)

		allGeneralTags = append(allGeneralTags, post.Tags.General...)
		allCharacterTags = append(allCharacterTags, post.Tags.Character...)
		allCopyrightTags = append(allCopyrightTags, post.Tags.Character...)
		allArtistTags = append(allArtistTags, post.Tags.Artist...)

		uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
		uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
		uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
		uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)

		// Uploads post to database
		err := neo4jAPI.CreatePostNode(ctx, driver, post.ID)
		if err != nil {
			return err
		}

		// Uploads the source to the database
		for _, source := range post.Sources {
			err := neo4jAPI.CreateSourceNode(ctx, driver, source)
			if err != nil {
				return err
			}
		}

		for _, uniqueGeneralTag := range uniqueGeneralTags {
			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
			if err != nil {
				return err
			}
		}
		for _, uniqueCharacterTag := range uniqueCharacterTags {
			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
			if err != nil {
				return err
			}
		}

		for _, uniqueCopyrightTag := range uniqueCopyrightTags {
			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
			if err != nil {
				return err
			}
		}

		for _, uniqueArtistTag := range uniqueArtistTags {
			err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
			if err != nil {
				return err
			}
		}
		log.Printf("Uploading post %d of %d with ID: %d took: %v", i, len(userFavorites), post.ID, time.Since(start))
	}

	for _, post := range userFavorites {
		id := post.ID

		err := neo4jAPI.UserToPostRelationship(ctx, driver, post.ID, e621User.ID)
		if err != nil {
			return err
		}
		log.Printf("Created UserToPostRelationship for user: %s to post: %d", username, post.ID)

		postID := post.ID
		for _, source := range post.Sources {
			err := neo4jAPI.PostToSourceRelationship(ctx, driver, postID, source)
			if err != nil {
				return err
			}
			log.Printf("Created PostToSourceRelationship for Post: %s to source: %d", post.ID, source)

		}

		for _, generalTag := range post.Tags.General {
			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, generalTag)
			if err != nil {
				log.Println(err)
			}
			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, generalTag)

		}
		for _, characterTag := range post.Tags.Character {
			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, characterTag)
			if err != nil {
				log.Println(err)
			}
			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, characterTag)

		}
		for _, copyrightTag := range post.Tags.Copyright {
			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, copyrightTag)
			if err != nil {
				log.Println(err)
			}
			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, copyrightTag)

		}
		for _, artistTag := range post.Tags.Artist {
			err := neo4jAPI.PostToTagRelationship(ctx, driver, id, artistTag)
			if err != nil {
				log.Println(err)
			}
			log.Printf("Created PostToTagRelationship for post: %d to tag: %s", post.ID, artistTag)

		}
	}

	log.Printf("Uploading posts for user %s took: %v", username, time.Since(startUploadPosts))
	return nil
}