e621-sdk-go/main.go

package main

import (
	"context"
	"e621_to_neo4j/e621/api"
	"e621_to_neo4j/e621/api/models"
	"e621_to_neo4j/neo4jAPI"
	"e621_to_neo4j/utils"
	"github.com/neo4j/neo4j-go-driver/v5/neo4j"
	"log"
	"time"
)

func main() {
	config, err := utils.LoadConfig()
	if err != nil {
		log.Println(err)
	}

	driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
	if err != nil {
		log.Fatal(err)
	}

	ctx := context.Background()
	defer driver.Close(ctx)

	e621Client := api.NewClient(config.E621APIKey, config.E621Username)
	user, err := e621Client.GetUserInfo("selloo")
	if err != nil {
		log.Fatal(err)
	}

	err = neo4jAPI.CreateUserNode(ctx, driver, user)
	if err != nil {
		log.Fatal(err)
	}

	favs, err := e621Client.GetFavorites(user.Name)
	if err != nil {
		log.Fatal(err)
	}

	start := time.Now()
	Upload3(favs, ctx, driver)
	elapsed := time.Since(start)

	log.Printf("This took %s", elapsed)
}

// ~4min
func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
	for i, fav := range favs {

		log.Printf("The e621 post with the id %d has %d general Tags, %d character Tags, %d copyright Tags, %d artist Tags.", fav.ID, len(fav.Tags.General), len(fav.Tags.Character), len(fav.Tags.Copyright), len(fav.Tags.Artist))
		log.Printf("Uploaded Posts: %d", i)

		for _, general := range fav.Tags.General {
			log.Printf("TagType: General - Tag: %s", general)
			err := neo4jAPI.CreateTagNode(ctx, driver, general, "general")
			if err != nil {
				log.Fatal(err)
			}
		}
		for _, character := range fav.Tags.Character {
			log.Printf("TagType: Character - Tag: %s", character)
			err := neo4jAPI.CreateTagNode(ctx, driver, character, "character")
			if err != nil {
				log.Fatal(err)
			}
		}

		for _, copyright := range fav.Tags.Copyright {
			log.Printf("TagType: Copyright - Tag: %s", copyright)
			err := neo4jAPI.CreateTagNode(ctx, driver, copyright, "copyright")
			if err != nil {
				log.Fatal(err)
			}
		}

		for _, artist := range fav.Tags.Artist {
			log.Printf("TagType: Artist - Tag: %s", artist)
			err := neo4jAPI.CreateTagNode(ctx, driver, artist, "artist")
			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

// ~30sec
func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
	uniqueGeneralTags := make(map[string]struct{})
	uniqueCharacterTags := make(map[string]struct{})
	uniqueCopyrightTags := make(map[string]struct{})
	uniqueArtistTags := make(map[string]struct{})

	allGeneralTags := make([]string, 0)
	allCharacterTags := make([]string, 0)
	allCopyrightTags := make([]string, 0)
	allArtistTags := make([]string, 0)

	for _, fav := range favs {

		// Process General Tags
		for _, general := range fav.Tags.General {
			uniqueGeneralTags[general] = struct{}{}
			allGeneralTags = append(allGeneralTags, general)
		}

		// Process Character Tags
		for _, character := range fav.Tags.Character {
			uniqueCharacterTags[character] = struct{}{}
			allCharacterTags = append(allCharacterTags, character)
		}

		// Process Copyright Tags
		for _, copyright := range fav.Tags.Copyright {
			uniqueCopyrightTags[copyright] = struct{}{}
			allCopyrightTags = append(allCopyrightTags, copyright)
		}

		// Process Artist Tags
		for _, artist := range fav.Tags.Artist {
			uniqueArtistTags[artist] = struct{}{}
			allArtistTags = append(allArtistTags, artist)
		}

	}

	log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
	log.Printf("allGeneralTags length: %d", len(allGeneralTags))

	log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
	log.Printf("allCharacterTags length: %d", len(allCharacterTags))

	log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
	log.Printf("alleCopyrightTags length: %d", len(allCopyrightTags))

	log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
	log.Printf("allArtistTags length: %d", len(allArtistTags))

	for uniqueGeneralTag := range uniqueGeneralTags {
		log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
		if err != nil {
			log.Fatal(err)
		}
	}
	for uniqueCharacterTag := range uniqueCharacterTags {
		log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
		if err != nil {
			log.Fatal(err)
		}
	}

	for uniqueCopyrightTag := range uniqueCopyrightTags {
		log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
		if err != nil {
			log.Fatal(err)
		}
	}

	for uniqueArtistTag := range uniqueArtistTags {
		log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
		if err != nil {
			log.Fatal(err)
		}
	}

}

// ~57s
func Upload3(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
	uniqueGeneralTags := make([]string, 0)
	uniqueCharacterTags := make([]string, 0)
	uniqueCopyrightTags := make([]string, 0)
	uniqueArtistTags := make([]string, 0)

	allGeneralTags := make([]string, 0)
	allCharacterTags := make([]string, 0)
	allCopyrightTags := make([]string, 0)
	allArtistTags := make([]string, 0)

	// add all tags together
	for _, fav := range favs {
		allGeneralTags = append(allGeneralTags, fav.Tags.General...)
		allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
		allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
		allArtistTags = append(allArtistTags, fav.Tags.Artist...)
	}

	// Process General Tags
	uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)

	// Process Character Tags
	uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)

	// Process Copyright Tags
	uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)

	// Process Artist Tags
	uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)

	log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))

	log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))

	log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))

	log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))

	for _, uniqueGeneralTag := range uniqueGeneralTags {

		log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
		if err != nil {
			log.Fatal(err)
		}
	}
	for _, uniqueCharacterTag := range uniqueCharacterTags {
		log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
		if err != nil {
			log.Fatal(err)
		}
	}

	for _, uniqueCopyrightTag := range uniqueCopyrightTags {
		log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
		if err != nil {
			log.Fatal(err)
		}
	}

	for _, uniqueArtistTag := range uniqueArtistTags {
		log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
		err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
		if err != nil {
			log.Fatal(err)
		}
	}

}