anthrove/e621-sdk-go
Archived
4
0
Fork 0
Code Issues 3 Pull Requests Packages Projects Releases Wiki Activity

nwo able to upload the sources to the DB

Browse Source
This commit is contained in:
David Janowski 2023-05-22 21:01:59 +02:00
parent 409788d0b0
commit 11dac75882
2 changed files with 30 additions and 86 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

110
main.go
View File

@ -23,7 +23,12 @@ func main() {
} }
ctx := context.Background() ctx := context.Background()
defer driver.Close(ctx) defer func(driver neo4j.DriverWithContext, ctx context.Context) {
err := driver.Close(ctx)
if err != nil {
log.Fatal(err)
}
}(driver, ctx)
e621Client := api.NewClient(config.E621APIKey, config.E621Username) e621Client := api.NewClient(config.E621APIKey, config.E621Username)
user, err := e621Client.GetUserInfo("selloo") user, err := e621Client.GetUserInfo("selloo")
@ -36,29 +41,30 @@ func main() {
log.Fatal(err) log.Fatal(err)
} }
favs, err := e621Client.GetFavorites(user.Name) favorites, err := e621Client.GetFavorites(user.Name)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
start := time.Now() start := time.Now()
uploadTags(favs, ctx, driver) uploadTags(favorites, ctx, driver)
elapsed := time.Since(start) elapsed := time.Since(start)
log.Printf("upload of Tags took %d", elapsed) log.Printf("upload of Tags took %d", elapsed)
start = time.Now() start = time.Now()
uploadPosts(favs, ctx, driver) uploadPosts(favorites, ctx, driver)
elapsed = time.Since(start) elapsed = time.Since(start)
log.Printf("upload of Posts took %d", elapsed) log.Printf("upload of Posts took %d", elapsed)
start = time.Now() start = time.Now()
//uploadSources(favs, ctx, driver) uploadSources(favorites, ctx, driver)
elapsed = time.Since(start) elapsed = time.Since(start)
log.Printf("upload of Sources took %d", elapsed) log.Printf("upload of Sources took %d", elapsed)
} }
func uploadPosts(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error { func uploadPosts(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
for _, fav := range favs { for _, fav := range posts {
log.Printf("e621 PostID: %d", fav.ID)
err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID) err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID)
if err != nil { if err != nil {
return err return err
@ -69,7 +75,7 @@ func uploadPosts(favs []models.Post, ctx context.Context, driver neo4j.DriverWit
} }
func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { func uploadTags(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
uniqueGeneralTags := make([]string, 0) uniqueGeneralTags := make([]string, 0)
uniqueCharacterTags := make([]string, 0) uniqueCharacterTags := make([]string, 0)
uniqueCopyrightTags := make([]string, 0) uniqueCopyrightTags := make([]string, 0)
@ -81,7 +87,7 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
allArtistTags := make([]string, 0) allArtistTags := make([]string, 0)
// add all tags together // add all tags together
for _, fav := range favs { for _, fav := range posts {
allGeneralTags = append(allGeneralTags, fav.Tags.General...) allGeneralTags = append(allGeneralTags, fav.Tags.General...)
allCharacterTags = append(allCharacterTags, fav.Tags.Character...) allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...) allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
@ -102,8 +108,6 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags)) log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags)) log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags)) log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
@ -113,14 +117,14 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag) log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general") err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
if err != nil { if err != nil {
log.Fatal(err) return err
} }
} }
for _, uniqueCharacterTag := range uniqueCharacterTags { for _, uniqueCharacterTag := range uniqueCharacterTags {
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag) log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character") err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
if err != nil { if err != nil {
log.Fatal(err) return err
} }
} }
@ -128,7 +132,7 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag) log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright") err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
if err != nil { if err != nil {
log.Fatal(err) return err
} }
} }
@ -136,81 +140,21 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag) log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist") err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
if err != nil { if err != nil {
log.Fatal(err) return err
} }
} }
return nil
} }
func uploadSources(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) { func uploadSources(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
uniqueGeneralTags := make([]string, 0) for _, fav := range posts {
uniqueCharacterTags := make([]string, 0) for _, source := range fav.Sources {
uniqueCopyrightTags := make([]string, 0) log.Printf("e621 Source: %s", source)
uniqueArtistTags := make([]string, 0) err := neo4jAPI.CreateSourceNode(ctx, driver, source)
allGeneralTags := make([]string, 0)
allCharacterTags := make([]string, 0)
allCopyrightTags := make([]string, 0)
allArtistTags := make([]string, 0)
// add all tags together
for _, fav := range favs {
allGeneralTags = append(allGeneralTags, fav.Tags.General...)
allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
allArtistTags = append(allArtistTags, fav.Tags.Artist...)
}
// Process General PostTags
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
// Process Character PostTags
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
// Process Copyright PostTags
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
// Process Artist PostTags
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
for _, uniqueGeneralTag := range uniqueGeneralTags {
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
if err != nil { if err != nil {
log.Fatal(err) return err
} }
} }
for _, uniqueCharacterTag := range uniqueCharacterTags {
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
if err != nil {
log.Fatal(err)
} }
} return nil
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
if err != nil {
log.Fatal(err)
}
}
for _, uniqueArtistTag := range uniqueArtistTags {
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
if err != nil {
log.Fatal(err)
}
}
} }

2
neo4jAPI/source.go
View File

@ -11,7 +11,7 @@ func CreateSourceNode(ctx context.Context, driver neo4j.DriverWithContext, URL s
RETURN u RETURN u
` `
params := map[string]any{ params := map[string]any{
"URL": URL, "url": URL,
} }
_, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer) _, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)