Added another method making tags unique

This commit is contained in:
David Janowski 2023-05-22 17:50:26 +02:00
parent 08502160cc
commit a7b97a0e35
2 changed files with 104 additions and 1 deletions

78
main.go
View File

@ -42,12 +42,13 @@ func main() {
}
start := time.Now()
Upload2(favs, ctx, driver)
Upload3(favs, ctx, driver)
elapsed := time.Since(start)
log.Printf("This took %s", elapsed)
}
// ~4min
func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
for i, fav := range favs {
@ -87,6 +88,7 @@ func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithCont
}
}
// ~30sec
func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
uniqueGeneralTags := make(map[string]struct{})
uniqueCharacterTags := make(map[string]struct{})
@ -170,3 +172,77 @@ func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithCon
}
}
// ~57s
func Upload3(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
uniqueGeneralTags := make([]string, 0)
uniqueCharacterTags := make([]string, 0)
uniqueCopyrightTags := make([]string, 0)
uniqueArtistTags := make([]string, 0)
allGeneralTags := make([]string, 0)
allCharacterTags := make([]string, 0)
allCopyrightTags := make([]string, 0)
allArtistTags := make([]string, 0)
// add all tags together
for _, fav := range favs {
allGeneralTags = append(allGeneralTags, fav.Tags.General...)
allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
allArtistTags = append(allArtistTags, fav.Tags.Artist...)
}
// Process General Tags
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
// Process Character Tags
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
// Process Copyright Tags
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
// Process Artist Tags
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
for _, uniqueGeneralTag := range uniqueGeneralTags {
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
if err != nil {
log.Fatal(err)
}
}
for _, uniqueCharacterTag := range uniqueCharacterTags {
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
if err != nil {
log.Fatal(err)
}
}
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
if err != nil {
log.Fatal(err)
}
}
for _, uniqueArtistTag := range uniqueArtistTags {
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
if err != nil {
log.Fatal(err)
}
}
}

27
utils/list.go Normal file
View File

@ -0,0 +1,27 @@
package utils
// UniqueNonEmptyElementsOf returns a new slice containing unique non-empty elements from the input slice.
// It removes duplicate elements and empty strings while preserving the order of appearance.
func UniqueNonEmptyElementsOf(s []string) []string {
// Create a map to store unique elements
unique := make(map[string]bool)
// Create a new slice to store the unique non-empty elements
us := make([]string, 0, len(s))
for _, elem := range s {
// Skip empty strings
if len(elem) == 0 {
continue
}
// Check if the element is already in the unique map
if !unique[elem] {
// Add the element to the unique map and the new slice
unique[elem] = true
us = append(us, elem)
}
}
return us
}