experiments uploading of the tags to neo4j
This commit is contained in:
parent
1dd14b9c82
commit
01fcb00191
110
main.go
110
main.go
@ -3,8 +3,10 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/api"
|
||||
"e621_to_neo4j/neo4j"
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"e621_to_neo4j/neo4jAPI"
|
||||
"e621_to_neo4j/utils"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
@ -15,7 +17,7 @@ func main() {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
driver, err := neo4j.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
|
||||
driver, err := neo4jAPI.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -29,7 +31,7 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = neo4j.CreateUserNode(ctx, driver, user)
|
||||
err = neo4jAPI.CreateUserNode(ctx, driver, user)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -38,7 +40,15 @@ func main() {
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
Upload2(favs, ctx, driver)
|
||||
elapsed := time.Since(start)
|
||||
|
||||
log.Printf("This took %s", elapsed)
|
||||
}
|
||||
|
||||
func Upload(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
|
||||
for i, fav := range favs {
|
||||
|
||||
log.Printf("The e621 post with the id %d has %d general Tags, %d character Tags, %d copyright Tags, %d artist Tags.", fav.ID, len(fav.Tags.General), len(fav.Tags.Character), len(fav.Tags.Copyright), len(fav.Tags.Artist))
|
||||
@ -46,14 +56,14 @@ func main() {
|
||||
|
||||
for _, general := range fav.Tags.General {
|
||||
log.Printf("TagType: General - Tag: %s", general)
|
||||
err := neo4j.CreateTagNode(ctx, driver, general, "general")
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, general, "general")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, character := range fav.Tags.Character {
|
||||
log.Printf("TagType: Character - Tag: %s", character)
|
||||
err := neo4j.CreateTagNode(ctx, driver, character, "character")
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, character, "character")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -61,7 +71,7 @@ func main() {
|
||||
|
||||
for _, copyright := range fav.Tags.Copyright {
|
||||
log.Printf("TagType: Copyright - Tag: %s", copyright)
|
||||
err := neo4j.CreateTagNode(ctx, driver, copyright, "copyright")
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, copyright, "copyright")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
@ -69,12 +79,94 @@ func main() {
|
||||
|
||||
for _, artist := range fav.Tags.Artist {
|
||||
log.Printf("TagType: Artist - Tag: %s", artist)
|
||||
err := neo4j.CreateTagNode(ctx, driver, artist, "artist")
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, artist, "artist")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("This took %s", elapsed)
|
||||
}
|
||||
|
||||
func Upload2(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
|
||||
uniqueGeneralTags := make(map[string]struct{})
|
||||
uniqueCharacterTags := make(map[string]struct{})
|
||||
uniqueCopyrightTags := make(map[string]struct{})
|
||||
uniqueArtistTags := make(map[string]struct{})
|
||||
|
||||
allGeneralTags := make([]string, 0)
|
||||
allCharacterTags := make([]string, 0)
|
||||
allCopyrightTags := make([]string, 0)
|
||||
allArtistTags := make([]string, 0)
|
||||
|
||||
for _, fav := range favs {
|
||||
|
||||
// Process General Tags
|
||||
for _, general := range fav.Tags.General {
|
||||
uniqueGeneralTags[general] = struct{}{}
|
||||
allGeneralTags = append(allGeneralTags, general)
|
||||
}
|
||||
|
||||
// Process Character Tags
|
||||
for _, character := range fav.Tags.Character {
|
||||
uniqueCharacterTags[character] = struct{}{}
|
||||
allCharacterTags = append(allCharacterTags, character)
|
||||
}
|
||||
|
||||
// Process Copyright Tags
|
||||
for _, copyright := range fav.Tags.Copyright {
|
||||
uniqueCopyrightTags[copyright] = struct{}{}
|
||||
allCopyrightTags = append(allCopyrightTags, copyright)
|
||||
}
|
||||
|
||||
// Process Artist Tags
|
||||
for _, artist := range fav.Tags.Artist {
|
||||
uniqueArtistTags[artist] = struct{}{}
|
||||
allArtistTags = append(allArtistTags, artist)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
|
||||
log.Printf("allGeneralTags length: %d", len(allGeneralTags))
|
||||
|
||||
log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
|
||||
log.Printf("allCharacterTags length: %d", len(allCharacterTags))
|
||||
|
||||
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
|
||||
log.Printf("alleCopyrightTags length: %d", len(allCopyrightTags))
|
||||
|
||||
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
|
||||
log.Printf("allArtistTags length: %d", len(allArtistTags))
|
||||
|
||||
for uniqueGeneralTag := range uniqueGeneralTags {
|
||||
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
for uniqueCharacterTag := range uniqueCharacterTags {
|
||||
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for uniqueCopyrightTag := range uniqueCopyrightTags {
|
||||
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for uniqueArtistTag := range uniqueArtistTags {
|
||||
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
Reference in New Issue
Block a user