From 0a5f281a1e04ed96cd827d25118fa0492225de12 Mon Sep 17 00:00:00 2001 From: SoXX Date: Tue, 2 Jul 2024 22:35:42 +0200 Subject: [PATCH] feat: added batch jobs for creating Tags, TagAliases & TagGroups Signed-off-by: SoXX --- internal/postgres/tag.go | 104 +++++++++++ internal/postgres/tag_test.go | 322 ++++++++++++++++++++++++++++++++++ pkg/database/postgres.go | 13 ++ pkg/database/postgres_test.go | 321 +++++++++++++++++++++++++++++++++ pkg/database/tag.go | 2 + pkg/database/tagalias.go | 2 + pkg/database/taggroup.go | 2 + 7 files changed, 766 insertions(+) diff --git a/internal/postgres/tag.go b/internal/postgres/tag.go index 94ae51d..bc7bef6 100644 --- a/internal/postgres/tag.go +++ b/internal/postgres/tag.go @@ -3,6 +3,7 @@ package postgres import ( "context" "errors" + "gorm.io/gorm/clause" otterError "git.dragse.it/anthrove/otter-space-sdk/v2/pkg/error" "git.dragse.it/anthrove/otter-space-sdk/v2/pkg/models" @@ -40,6 +41,43 @@ func CreateTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName, return nil } +func CreateTagInBatchAndUpdate(ctx context.Context, db *gorm.DB, tags []models.Tag, batchSize int) error { + if len(tags) == 0 { + return &otterError.EntityValidationFailed{Reason: "tags cannot be empty"} + } + + if tags == nil { + return &otterError.EntityValidationFailed{Reason: "tags cannot be nil"} + } + + if batchSize == 0 { + return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"} + } + + result := db.WithContext(ctx). + Clauses(clause.OnConflict{ + Columns: []clause.Column{{Name: "name"}}, + DoUpdates: clause.AssignmentColumns([]string{"tag_type"}), + }).CreateInBatches(tags, batchSize) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrDuplicatedKey) { + return &otterError.EntityAlreadyExists{} + } + return result.Error + } + + if result.RowsAffected == 0 { + return &otterError.NoDataWritten{} + } + + log.WithFields(log.Fields{ + "tag_size": len(tags), + "batch_size": batchSize, + }).Trace("database: created tag node") + + return nil +} + func DeleteTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName) error { if tagName == "" { @@ -169,6 +207,39 @@ func CreateTagAlias(ctx context.Context, db *gorm.DB, tagAliasName models.Anthro return nil } +func CreateTagAliasInBatch(ctx context.Context, db *gorm.DB, tagAliases []models.TagAlias, batchSize int) error { + if len(tagAliases) == 0 { + return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"} + } + + if tagAliases == nil { + return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"} + } + + if batchSize == 0 { + return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"} + } + + result := db.WithContext(ctx).CreateInBatches(tagAliases, batchSize) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrDuplicatedKey) { + return &otterError.EntityAlreadyExists{} + } + return result.Error + } + + if result.RowsAffected == 0 { + return &otterError.NoDataWritten{} + } + + log.WithFields(log.Fields{ + "tag_size": len(tagAliases), + "batch_size": batchSize, + }).Trace("database: created tag node") + + return nil +} + func GetAllTagAlias(ctx context.Context, db *gorm.DB) ([]models.TagAlias, error) { var tagAliases []models.TagAlias @@ -263,6 +334,39 @@ func CreateTagGroup(ctx context.Context, db *gorm.DB, tagGroupName models.Anthro return nil } +func CreateTagGroupInBatch(ctx context.Context, db *gorm.DB, tagGroups []models.TagGroup, batchSize int) error { + if len(tagGroups) == 0 { + return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"} + } + + if tagGroups == nil { + return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"} + } + + if batchSize == 0 { + return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"} + } + + result := db.WithContext(ctx).CreateInBatches(tagGroups, batchSize) + if result.Error != nil { + if errors.Is(result.Error, gorm.ErrDuplicatedKey) { + return &otterError.EntityAlreadyExists{} + } + return result.Error + } + + if result.RowsAffected == 0 { + return &otterError.NoDataWritten{} + } + + log.WithFields(log.Fields{ + "tag_size": len(tagGroups), + "batch_size": batchSize, + }).Trace("database: created tag node") + + return nil +} + func GetAllTagGroup(ctx context.Context, db *gorm.DB) ([]models.TagGroup, error) { var tagGroups []models.TagGroup diff --git a/internal/postgres/tag_test.go b/internal/postgres/tag_test.go index 9c6891d..179849e 100644 --- a/internal/postgres/tag_test.go +++ b/internal/postgres/tag_test.go @@ -1128,3 +1128,325 @@ func TestGetAllTagByTagType(t *testing.T) { }) } } + +func TestCreateTagInBatchAndUpdate(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + t.Fatalf("Could not start PostgreSQL container: %v", err) + } + defer container.Terminate(ctx) + + // Setup Test + tags := []models.Tag{ + { + Name: "JayTheFerret", + Type: models.Artist, + }, + { + Name: "SoXX", + Type: models.Character, + }, + { + Name: "Dragon", + Type: models.Species, + }, + { + Name: "Fennec", + Type: models.Species, + }, + } + emptyTags := []models.Tag{} + + // Test + type args struct { + ctx context.Context + db *gorm.DB + tags []models.Tag + batchSize int + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Test 1: Valid Tags", + args: args{ + ctx: ctx, + db: gormDB, + tags: tags, + batchSize: 10, + }, + wantErr: false, + }, + { + name: "Test 2: Empty Tags", + args: args{ + ctx: ctx, + db: gormDB, + tags: emptyTags, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 3: Nil Tags", + args: args{ + ctx: ctx, + db: gormDB, + tags: nil, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 4: No batchSize", + args: args{ + ctx: ctx, + db: gormDB, + tags: nil, + batchSize: 0, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.db, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr { + t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestCreateTagAliasInBatch(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + t.Fatalf("Could not start PostgreSQL container: %v", err) + } + defer container.Terminate(ctx) + + // Setup Test + tags := []models.Tag{ + { + Name: "JayTheFerret", + Type: models.Artist, + }, + { + Name: "SoXX", + Type: models.Character, + }, + { + Name: "Dragon", + Type: models.Species, + }, + { + Name: "Fennec", + Type: models.Species, + }, + } + err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags)) + if err != nil { + t.Fatal(err) + } + + tagAlias := []models.TagAlias{ + { + Name: "test1", + TagID: tags[0].Name, + }, + { + Name: "test2", + TagID: tags[1].Name, + }, + { + Name: "test3", + TagID: tags[2].Name, + }, + { + Name: "test4", + TagID: tags[3].Name, + }, + } + emptyTagAlias := []models.TagAlias{} + + // Test + type args struct { + ctx context.Context + db *gorm.DB + tagAliases []models.TagAlias + batchSize int + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Test 1: Valid Tags", + args: args{ + ctx: ctx, + db: gormDB, + tagAliases: tagAlias, + batchSize: 10, + }, + wantErr: false, + }, + { + name: "Test 2: Empty Tags", + args: args{ + ctx: ctx, + db: gormDB, + tagAliases: emptyTagAlias, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 3: Nil Tags", + args: args{ + ctx: ctx, + db: gormDB, + tagAliases: nil, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 4: No batchSize", + args: args{ + ctx: ctx, + db: gormDB, + tagAliases: tagAlias, + batchSize: 0, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CreateTagAliasInBatch(tt.args.ctx, tt.args.db, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr { + t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestCreateTagGroupInBatch(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + t.Fatalf("Could not start PostgreSQL container: %v", err) + } + defer container.Terminate(ctx) + + // Setup Test + tags := []models.Tag{ + { + Name: "JayTheFerret", + Type: models.Artist, + }, + { + Name: "SoXX", + Type: models.Character, + }, + { + Name: "Dragon", + Type: models.Species, + }, + { + Name: "Fennec", + Type: models.Species, + }, + } + err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags)) + if err != nil { + t.Fatal(err) + } + + tagGroup := []models.TagGroup{ + { + Name: "test1", + TagID: tags[0].Name, + }, + { + Name: "test2", + TagID: tags[1].Name, + }, + { + Name: "test3", + TagID: tags[2].Name, + }, + { + Name: "test4", + TagID: tags[3].Name, + }, + } + emptyTagGroup := []models.TagGroup{} + + // Test + type args struct { + ctx context.Context + db *gorm.DB + tagGroups []models.TagGroup + batchSize int + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Test 1: Valid Tags", + args: args{ + ctx: ctx, + db: gormDB, + tagGroups: tagGroup, + batchSize: 10, + }, + wantErr: false, + }, + { + name: "Test 2: Empty Tags", + args: args{ + ctx: ctx, + db: gormDB, + tagGroups: emptyTagGroup, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 3: Nil Tags", + args: args{ + ctx: ctx, + db: gormDB, + tagGroups: nil, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 4: No batchSize", + args: args{ + ctx: ctx, + db: gormDB, + tagGroups: tagGroup, + batchSize: 0, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := CreateTagGroupInBatch(tt.args.ctx, tt.args.db, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr { + t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/database/postgres.go b/pkg/database/postgres.go index ad47bf2..08750d6 100644 --- a/pkg/database/postgres.go +++ b/pkg/database/postgres.go @@ -214,6 +214,19 @@ func (p *postgresqlConnection) DeleteTag(ctx context.Context, tagName models.Ant return postgres.DeleteTag(ctx, p.db, tagName) } +func (p *postgresqlConnection) CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error { + return postgres.CreateTagInBatchAndUpdate(ctx, p.db, tags, batchSize) +} + +func (p *postgresqlConnection) CreateTagAliasInBatch(ctx context.Context, tagAliases []models.TagAlias, batchSize int) error { + return postgres.CreateTagAliasInBatch(ctx, p.db, tagAliases, batchSize) +} + +func (p *postgresqlConnection) CreateTagGroupInBatch(ctx context.Context, tagGroups []models.TagGroup, batchSize int) error { + return postgres.CreateTagGroupInBatch(ctx, p.db, tagGroups, batchSize) + +} + // HELPER func (p *postgresqlConnection) migrateDatabase(dbPool *gorm.DB) error { diff --git a/pkg/database/postgres_test.go b/pkg/database/postgres_test.go index 553caac..b48f514 100644 --- a/pkg/database/postgres_test.go +++ b/pkg/database/postgres_test.go @@ -3252,3 +3252,324 @@ func Test_postgresqlConnection_GetAllTagsByTagType(t *testing.T) { }) } } + +func Test_postgresqlConnection_CreateTagInBatchAndUpdate(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + t.Fatalf("Could not start PostgreSQL container: %v", err) + } + defer container.Terminate(ctx) + + // Setup Test + tags := []models.Tag{ + { + Name: "JayTheFerret", + Type: models.Artist, + }, + { + Name: "SoXX", + Type: models.Character, + }, + { + Name: "Dragon", + Type: models.Species, + }, + { + Name: "Fennec", + Type: models.Species, + }, + } + emptyTags := []models.Tag{} + + // Test + type args struct { + ctx context.Context + tags []models.Tag + batchSize int + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Test 1: Valid Tags", + args: args{ + ctx: ctx, + tags: tags, + batchSize: 10, + }, + wantErr: false, + }, + { + name: "Test 2: Empty Tags", + args: args{ + ctx: ctx, + tags: emptyTags, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 3: Nil Tags", + args: args{ + ctx: ctx, + tags: nil, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 4: No batchSize", + args: args{ + ctx: ctx, + tags: nil, + batchSize: 0, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &postgresqlConnection{ + db: gormDB, + debug: true, + } + if err := p.CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr { + t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_postgresqlConnection_CreateTagAliasInBatch(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + t.Fatalf("Could not start PostgreSQL container: %v", err) + } + defer container.Terminate(ctx) + + // Setup Test + tags := []models.Tag{ + { + Name: "JayTheFerret", + Type: models.Artist, + }, + { + Name: "SoXX", + Type: models.Character, + }, + { + Name: "Dragon", + Type: models.Species, + }, + { + Name: "Fennec", + Type: models.Species, + }, + } + err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags)) + if err != nil { + t.Fatal(err) + } + + tagAlias := []models.TagAlias{ + { + Name: "test1", + TagID: tags[0].Name, + }, + { + Name: "test2", + TagID: tags[1].Name, + }, + { + Name: "test3", + TagID: tags[2].Name, + }, + { + Name: "test4", + TagID: tags[3].Name, + }, + } + emptyTagAlias := []models.TagAlias{} + + // Test + type args struct { + ctx context.Context + tagAliases []models.TagAlias + batchSize int + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Test 1: Valid Tags", + args: args{ + ctx: ctx, + tagAliases: tagAlias, + batchSize: 10, + }, + wantErr: false, + }, + { + name: "Test 2: Empty Tags", + args: args{ + ctx: ctx, + tagAliases: emptyTagAlias, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 3: Nil Tags", + args: args{ + ctx: ctx, + tagAliases: nil, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 4: No batchSize", + args: args{ + ctx: ctx, + tagAliases: tagAlias, + batchSize: 0, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &postgresqlConnection{ + db: gormDB, + debug: true, + } + if err := p.CreateTagAliasInBatch(tt.args.ctx, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr { + t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_postgresqlConnection_CreateTagGroupInBatch(t *testing.T) { + // Setup trow away container + ctx := context.Background() + container, gormDB, err := test.StartPostgresContainer(ctx) + if err != nil { + t.Fatalf("Could not start PostgreSQL container: %v", err) + } + defer container.Terminate(ctx) + + // Setup Test + tags := []models.Tag{ + { + Name: "JayTheFerret", + Type: models.Artist, + }, + { + Name: "SoXX", + Type: models.Character, + }, + { + Name: "Dragon", + Type: models.Species, + }, + { + Name: "Fennec", + Type: models.Species, + }, + } + err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags)) + if err != nil { + t.Fatal(err) + } + + tagGroup := []models.TagGroup{ + { + Name: "test1", + TagID: tags[0].Name, + }, + { + Name: "test2", + TagID: tags[1].Name, + }, + { + Name: "test3", + TagID: tags[2].Name, + }, + { + Name: "test4", + TagID: tags[3].Name, + }, + } + emptyTagGroup := []models.TagGroup{} + + // Test + type args struct { + ctx context.Context + tagGroups []models.TagGroup + batchSize int + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "Test 1: Valid Tags", + args: args{ + ctx: ctx, + tagGroups: tagGroup, + batchSize: 10, + }, + wantErr: false, + }, + { + name: "Test 2: Empty Tags", + args: args{ + ctx: ctx, + tagGroups: emptyTagGroup, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 3: Nil Tags", + args: args{ + ctx: ctx, + tagGroups: nil, + batchSize: 10, + }, + wantErr: true, + }, + { + name: "Test 4: No batchSize", + args: args{ + ctx: ctx, + tagGroups: tagGroup, + batchSize: 0, + }, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + p := &postgresqlConnection{ + db: gormDB, + debug: true, + } + if err := p.CreateTagGroupInBatch(tt.args.ctx, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr { + t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/pkg/database/tag.go b/pkg/database/tag.go index b034f2f..b9b3add 100644 --- a/pkg/database/tag.go +++ b/pkg/database/tag.go @@ -9,6 +9,8 @@ import ( type Tag interface { CreateTag(ctx context.Context, tagName models.AnthroveTagName, tagType models.TagType) error + CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error + // GetAllTags retrieves all tags. GetAllTags(ctx context.Context) ([]models.Tag, error) diff --git a/pkg/database/tagalias.go b/pkg/database/tagalias.go index 0aeba5a..aeaca57 100644 --- a/pkg/database/tagalias.go +++ b/pkg/database/tagalias.go @@ -9,6 +9,8 @@ import ( type TagAlias interface { CreateTagAlias(ctx context.Context, tagAliasName models.AnthroveTagAliasName, tagID models.AnthroveTagID) error + CreateTagAliasInBatch(ctx context.Context, tagsAliases []models.TagAlias, batchSize int) error + GetAllTagAlias(ctx context.Context) ([]models.TagAlias, error) GetAllTagAliasByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagAlias, error) diff --git a/pkg/database/taggroup.go b/pkg/database/taggroup.go index b62e28d..baef4a1 100644 --- a/pkg/database/taggroup.go +++ b/pkg/database/taggroup.go @@ -9,6 +9,8 @@ import ( type TagGroup interface { CreateTagGroup(ctx context.Context, tagGroupName models.AnthroveTagGroupName, tagID models.AnthroveTagID) error + CreateTagGroupInBatch(ctx context.Context, tagsGroups []models.TagGroup, batchSize int) error + GetAllTagGroup(ctx context.Context) ([]models.TagGroup, error) GetAllTagGroupByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagGroup, error)