nwo able to upload the sources to the DB
This commit is contained in:
parent
409788d0b0
commit
11dac75882
114
main.go
114
main.go
@ -23,7 +23,12 @@ func main() {
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
defer driver.Close(ctx)
|
||||
defer func(driver neo4j.DriverWithContext, ctx context.Context) {
|
||||
err := driver.Close(ctx)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}(driver, ctx)
|
||||
|
||||
e621Client := api.NewClient(config.E621APIKey, config.E621Username)
|
||||
user, err := e621Client.GetUserInfo("selloo")
|
||||
@ -36,29 +41,30 @@ func main() {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
favs, err := e621Client.GetFavorites(user.Name)
|
||||
favorites, err := e621Client.GetFavorites(user.Name)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
start := time.Now()
|
||||
uploadTags(favs, ctx, driver)
|
||||
uploadTags(favorites, ctx, driver)
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("upload of Tags took %d", elapsed)
|
||||
|
||||
start = time.Now()
|
||||
uploadPosts(favs, ctx, driver)
|
||||
uploadPosts(favorites, ctx, driver)
|
||||
elapsed = time.Since(start)
|
||||
log.Printf("upload of Posts took %d", elapsed)
|
||||
|
||||
start = time.Now()
|
||||
//uploadSources(favs, ctx, driver)
|
||||
uploadSources(favorites, ctx, driver)
|
||||
elapsed = time.Since(start)
|
||||
log.Printf("upload of Sources took %d", elapsed)
|
||||
|
||||
}
|
||||
func uploadPosts(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
for _, fav := range favs {
|
||||
func uploadPosts(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
for _, fav := range posts {
|
||||
log.Printf("e621 PostID: %d", fav.ID)
|
||||
err := neo4jAPI.CreatePostNode(ctx, driver, fav.ID)
|
||||
if err != nil {
|
||||
return err
|
||||
@ -69,7 +75,7 @@ func uploadPosts(favs []models.Post, ctx context.Context, driver neo4j.DriverWit
|
||||
|
||||
}
|
||||
|
||||
func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
|
||||
func uploadTags(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
uniqueGeneralTags := make([]string, 0)
|
||||
uniqueCharacterTags := make([]string, 0)
|
||||
uniqueCopyrightTags := make([]string, 0)
|
||||
@ -81,7 +87,7 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
|
||||
allArtistTags := make([]string, 0)
|
||||
|
||||
// add all tags together
|
||||
for _, fav := range favs {
|
||||
for _, fav := range posts {
|
||||
allGeneralTags = append(allGeneralTags, fav.Tags.General...)
|
||||
allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
|
||||
allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
|
||||
@ -102,8 +108,6 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
|
||||
|
||||
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
|
||||
|
||||
log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
|
||||
|
||||
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
|
||||
|
||||
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
|
||||
@ -113,14 +117,14 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
|
||||
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, uniqueCharacterTag := range uniqueCharacterTags {
|
||||
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@ -128,7 +132,7 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
|
||||
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
@ -136,81 +140,21 @@ func uploadTags(favs []models.Post, ctx context.Context, driver neo4j.DriverWith
|
||||
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func uploadSources(favs []models.Post, ctx context.Context, driver neo4j.DriverWithContext) {
|
||||
uniqueGeneralTags := make([]string, 0)
|
||||
uniqueCharacterTags := make([]string, 0)
|
||||
uniqueCopyrightTags := make([]string, 0)
|
||||
uniqueArtistTags := make([]string, 0)
|
||||
|
||||
allGeneralTags := make([]string, 0)
|
||||
allCharacterTags := make([]string, 0)
|
||||
allCopyrightTags := make([]string, 0)
|
||||
allArtistTags := make([]string, 0)
|
||||
|
||||
// add all tags together
|
||||
for _, fav := range favs {
|
||||
allGeneralTags = append(allGeneralTags, fav.Tags.General...)
|
||||
allCharacterTags = append(allCharacterTags, fav.Tags.Character...)
|
||||
allCopyrightTags = append(allCopyrightTags, fav.Tags.Character...)
|
||||
allArtistTags = append(allArtistTags, fav.Tags.Artist...)
|
||||
}
|
||||
|
||||
// Process General PostTags
|
||||
uniqueGeneralTags = utils.UniqueNonEmptyElementsOf(allGeneralTags)
|
||||
|
||||
// Process Character PostTags
|
||||
uniqueCharacterTags = utils.UniqueNonEmptyElementsOf(allCharacterTags)
|
||||
|
||||
// Process Copyright PostTags
|
||||
uniqueCopyrightTags = utils.UniqueNonEmptyElementsOf(allCopyrightTags)
|
||||
|
||||
// Process Artist PostTags
|
||||
uniqueArtistTags = utils.UniqueNonEmptyElementsOf(allArtistTags)
|
||||
|
||||
log.Printf("uniqueGeneralTags length: %d", len(uniqueGeneralTags))
|
||||
|
||||
log.Printf("uniqueCharacterTags length: %d", len(uniqueCharacterTags))
|
||||
|
||||
log.Printf("uniqueCopyrightTags length: %d", len(uniqueCopyrightTags))
|
||||
|
||||
log.Printf("uniqueArtistTags length: %d", len(uniqueArtistTags))
|
||||
|
||||
for _, uniqueGeneralTag := range uniqueGeneralTags {
|
||||
|
||||
log.Printf("TagType: General - Tag: %s", uniqueGeneralTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueGeneralTag, "general")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
func uploadSources(posts []models.Post, ctx context.Context, driver neo4j.DriverWithContext) error {
|
||||
for _, fav := range posts {
|
||||
for _, source := range fav.Sources {
|
||||
log.Printf("e621 Source: %s", source)
|
||||
err := neo4jAPI.CreateSourceNode(ctx, driver, source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, uniqueCharacterTag := range uniqueCharacterTags {
|
||||
log.Printf("TagType: Character - Tag: %s", uniqueCharacterTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCharacterTag, "character")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueCopyrightTag := range uniqueCopyrightTags {
|
||||
log.Printf("TagType: Copyright - Tag: %s", uniqueCopyrightTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueCopyrightTag, "copyright")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, uniqueArtistTag := range uniqueArtistTags {
|
||||
log.Printf("TagType: Artist - Tag: %s", uniqueArtistTag)
|
||||
err := neo4jAPI.CreateTagNode(ctx, driver, uniqueArtistTag, "artist")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
@ -11,7 +11,7 @@ func CreateSourceNode(ctx context.Context, driver neo4j.DriverWithContext, URL s
|
||||
RETURN u
|
||||
`
|
||||
params := map[string]any{
|
||||
"URL": URL,
|
||||
"url": URL,
|
||||
}
|
||||
|
||||
_, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)
|
||||
|
Reference in New Issue
Block a user