feat: added batch jobs for creating Tags, TagAliases & TagGroups

Signed-off-by: SoXX <soxx@fenpa.ws>
This commit is contained in:
SoXX 2024-07-02 22:35:42 +02:00
parent 5784b469b9
commit 0a5f281a1e
7 changed files with 766 additions and 0 deletions

View File

@ -3,6 +3,7 @@ package postgres
import ( import (
"context" "context"
"errors" "errors"
"gorm.io/gorm/clause"
otterError "git.dragse.it/anthrove/otter-space-sdk/v2/pkg/error" otterError "git.dragse.it/anthrove/otter-space-sdk/v2/pkg/error"
"git.dragse.it/anthrove/otter-space-sdk/v2/pkg/models" "git.dragse.it/anthrove/otter-space-sdk/v2/pkg/models"
@ -40,6 +41,43 @@ func CreateTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName,
return nil return nil
} }
func CreateTagInBatchAndUpdate(ctx context.Context, db *gorm.DB, tags []models.Tag, batchSize int) error {
if len(tags) == 0 {
return &otterError.EntityValidationFailed{Reason: "tags cannot be empty"}
}
if tags == nil {
return &otterError.EntityValidationFailed{Reason: "tags cannot be nil"}
}
if batchSize == 0 {
return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
}
result := db.WithContext(ctx).
Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "name"}},
DoUpdates: clause.AssignmentColumns([]string{"tag_type"}),
}).CreateInBatches(tags, batchSize)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
return &otterError.EntityAlreadyExists{}
}
return result.Error
}
if result.RowsAffected == 0 {
return &otterError.NoDataWritten{}
}
log.WithFields(log.Fields{
"tag_size": len(tags),
"batch_size": batchSize,
}).Trace("database: created tag node")
return nil
}
func DeleteTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName) error { func DeleteTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName) error {
if tagName == "" { if tagName == "" {
@ -169,6 +207,39 @@ func CreateTagAlias(ctx context.Context, db *gorm.DB, tagAliasName models.Anthro
return nil return nil
} }
func CreateTagAliasInBatch(ctx context.Context, db *gorm.DB, tagAliases []models.TagAlias, batchSize int) error {
if len(tagAliases) == 0 {
return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"}
}
if tagAliases == nil {
return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"}
}
if batchSize == 0 {
return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
}
result := db.WithContext(ctx).CreateInBatches(tagAliases, batchSize)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
return &otterError.EntityAlreadyExists{}
}
return result.Error
}
if result.RowsAffected == 0 {
return &otterError.NoDataWritten{}
}
log.WithFields(log.Fields{
"tag_size": len(tagAliases),
"batch_size": batchSize,
}).Trace("database: created tag node")
return nil
}
func GetAllTagAlias(ctx context.Context, db *gorm.DB) ([]models.TagAlias, error) { func GetAllTagAlias(ctx context.Context, db *gorm.DB) ([]models.TagAlias, error) {
var tagAliases []models.TagAlias var tagAliases []models.TagAlias
@ -263,6 +334,39 @@ func CreateTagGroup(ctx context.Context, db *gorm.DB, tagGroupName models.Anthro
return nil return nil
} }
func CreateTagGroupInBatch(ctx context.Context, db *gorm.DB, tagGroups []models.TagGroup, batchSize int) error {
if len(tagGroups) == 0 {
return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"}
}
if tagGroups == nil {
return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"}
}
if batchSize == 0 {
return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
}
result := db.WithContext(ctx).CreateInBatches(tagGroups, batchSize)
if result.Error != nil {
if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
return &otterError.EntityAlreadyExists{}
}
return result.Error
}
if result.RowsAffected == 0 {
return &otterError.NoDataWritten{}
}
log.WithFields(log.Fields{
"tag_size": len(tagGroups),
"batch_size": batchSize,
}).Trace("database: created tag node")
return nil
}
func GetAllTagGroup(ctx context.Context, db *gorm.DB) ([]models.TagGroup, error) { func GetAllTagGroup(ctx context.Context, db *gorm.DB) ([]models.TagGroup, error) {
var tagGroups []models.TagGroup var tagGroups []models.TagGroup

View File

@ -1128,3 +1128,325 @@ func TestGetAllTagByTagType(t *testing.T) {
}) })
} }
} }
func TestCreateTagInBatchAndUpdate(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
t.Fatalf("Could not start PostgreSQL container: %v", err)
}
defer container.Terminate(ctx)
// Setup Test
tags := []models.Tag{
{
Name: "JayTheFerret",
Type: models.Artist,
},
{
Name: "SoXX",
Type: models.Character,
},
{
Name: "Dragon",
Type: models.Species,
},
{
Name: "Fennec",
Type: models.Species,
},
}
emptyTags := []models.Tag{}
// Test
type args struct {
ctx context.Context
db *gorm.DB
tags []models.Tag
batchSize int
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Test 1: Valid Tags",
args: args{
ctx: ctx,
db: gormDB,
tags: tags,
batchSize: 10,
},
wantErr: false,
},
{
name: "Test 2: Empty Tags",
args: args{
ctx: ctx,
db: gormDB,
tags: emptyTags,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 3: Nil Tags",
args: args{
ctx: ctx,
db: gormDB,
tags: nil,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 4: No batchSize",
args: args{
ctx: ctx,
db: gormDB,
tags: nil,
batchSize: 0,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.db, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr {
t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestCreateTagAliasInBatch(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
t.Fatalf("Could not start PostgreSQL container: %v", err)
}
defer container.Terminate(ctx)
// Setup Test
tags := []models.Tag{
{
Name: "JayTheFerret",
Type: models.Artist,
},
{
Name: "SoXX",
Type: models.Character,
},
{
Name: "Dragon",
Type: models.Species,
},
{
Name: "Fennec",
Type: models.Species,
},
}
err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
if err != nil {
t.Fatal(err)
}
tagAlias := []models.TagAlias{
{
Name: "test1",
TagID: tags[0].Name,
},
{
Name: "test2",
TagID: tags[1].Name,
},
{
Name: "test3",
TagID: tags[2].Name,
},
{
Name: "test4",
TagID: tags[3].Name,
},
}
emptyTagAlias := []models.TagAlias{}
// Test
type args struct {
ctx context.Context
db *gorm.DB
tagAliases []models.TagAlias
batchSize int
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Test 1: Valid Tags",
args: args{
ctx: ctx,
db: gormDB,
tagAliases: tagAlias,
batchSize: 10,
},
wantErr: false,
},
{
name: "Test 2: Empty Tags",
args: args{
ctx: ctx,
db: gormDB,
tagAliases: emptyTagAlias,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 3: Nil Tags",
args: args{
ctx: ctx,
db: gormDB,
tagAliases: nil,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 4: No batchSize",
args: args{
ctx: ctx,
db: gormDB,
tagAliases: tagAlias,
batchSize: 0,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := CreateTagAliasInBatch(tt.args.ctx, tt.args.db, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr {
t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func TestCreateTagGroupInBatch(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
t.Fatalf("Could not start PostgreSQL container: %v", err)
}
defer container.Terminate(ctx)
// Setup Test
tags := []models.Tag{
{
Name: "JayTheFerret",
Type: models.Artist,
},
{
Name: "SoXX",
Type: models.Character,
},
{
Name: "Dragon",
Type: models.Species,
},
{
Name: "Fennec",
Type: models.Species,
},
}
err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
if err != nil {
t.Fatal(err)
}
tagGroup := []models.TagGroup{
{
Name: "test1",
TagID: tags[0].Name,
},
{
Name: "test2",
TagID: tags[1].Name,
},
{
Name: "test3",
TagID: tags[2].Name,
},
{
Name: "test4",
TagID: tags[3].Name,
},
}
emptyTagGroup := []models.TagGroup{}
// Test
type args struct {
ctx context.Context
db *gorm.DB
tagGroups []models.TagGroup
batchSize int
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Test 1: Valid Tags",
args: args{
ctx: ctx,
db: gormDB,
tagGroups: tagGroup,
batchSize: 10,
},
wantErr: false,
},
{
name: "Test 2: Empty Tags",
args: args{
ctx: ctx,
db: gormDB,
tagGroups: emptyTagGroup,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 3: Nil Tags",
args: args{
ctx: ctx,
db: gormDB,
tagGroups: nil,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 4: No batchSize",
args: args{
ctx: ctx,
db: gormDB,
tagGroups: tagGroup,
batchSize: 0,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if err := CreateTagGroupInBatch(tt.args.ctx, tt.args.db, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr {
t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@ -214,6 +214,19 @@ func (p *postgresqlConnection) DeleteTag(ctx context.Context, tagName models.Ant
return postgres.DeleteTag(ctx, p.db, tagName) return postgres.DeleteTag(ctx, p.db, tagName)
} }
func (p *postgresqlConnection) CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error {
return postgres.CreateTagInBatchAndUpdate(ctx, p.db, tags, batchSize)
}
func (p *postgresqlConnection) CreateTagAliasInBatch(ctx context.Context, tagAliases []models.TagAlias, batchSize int) error {
return postgres.CreateTagAliasInBatch(ctx, p.db, tagAliases, batchSize)
}
func (p *postgresqlConnection) CreateTagGroupInBatch(ctx context.Context, tagGroups []models.TagGroup, batchSize int) error {
return postgres.CreateTagGroupInBatch(ctx, p.db, tagGroups, batchSize)
}
// HELPER // HELPER
func (p *postgresqlConnection) migrateDatabase(dbPool *gorm.DB) error { func (p *postgresqlConnection) migrateDatabase(dbPool *gorm.DB) error {

View File

@ -3252,3 +3252,324 @@ func Test_postgresqlConnection_GetAllTagsByTagType(t *testing.T) {
}) })
} }
} }
func Test_postgresqlConnection_CreateTagInBatchAndUpdate(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
t.Fatalf("Could not start PostgreSQL container: %v", err)
}
defer container.Terminate(ctx)
// Setup Test
tags := []models.Tag{
{
Name: "JayTheFerret",
Type: models.Artist,
},
{
Name: "SoXX",
Type: models.Character,
},
{
Name: "Dragon",
Type: models.Species,
},
{
Name: "Fennec",
Type: models.Species,
},
}
emptyTags := []models.Tag{}
// Test
type args struct {
ctx context.Context
tags []models.Tag
batchSize int
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Test 1: Valid Tags",
args: args{
ctx: ctx,
tags: tags,
batchSize: 10,
},
wantErr: false,
},
{
name: "Test 2: Empty Tags",
args: args{
ctx: ctx,
tags: emptyTags,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 3: Nil Tags",
args: args{
ctx: ctx,
tags: nil,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 4: No batchSize",
args: args{
ctx: ctx,
tags: nil,
batchSize: 0,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &postgresqlConnection{
db: gormDB,
debug: true,
}
if err := p.CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr {
t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func Test_postgresqlConnection_CreateTagAliasInBatch(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
t.Fatalf("Could not start PostgreSQL container: %v", err)
}
defer container.Terminate(ctx)
// Setup Test
tags := []models.Tag{
{
Name: "JayTheFerret",
Type: models.Artist,
},
{
Name: "SoXX",
Type: models.Character,
},
{
Name: "Dragon",
Type: models.Species,
},
{
Name: "Fennec",
Type: models.Species,
},
}
err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
if err != nil {
t.Fatal(err)
}
tagAlias := []models.TagAlias{
{
Name: "test1",
TagID: tags[0].Name,
},
{
Name: "test2",
TagID: tags[1].Name,
},
{
Name: "test3",
TagID: tags[2].Name,
},
{
Name: "test4",
TagID: tags[3].Name,
},
}
emptyTagAlias := []models.TagAlias{}
// Test
type args struct {
ctx context.Context
tagAliases []models.TagAlias
batchSize int
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Test 1: Valid Tags",
args: args{
ctx: ctx,
tagAliases: tagAlias,
batchSize: 10,
},
wantErr: false,
},
{
name: "Test 2: Empty Tags",
args: args{
ctx: ctx,
tagAliases: emptyTagAlias,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 3: Nil Tags",
args: args{
ctx: ctx,
tagAliases: nil,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 4: No batchSize",
args: args{
ctx: ctx,
tagAliases: tagAlias,
batchSize: 0,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &postgresqlConnection{
db: gormDB,
debug: true,
}
if err := p.CreateTagAliasInBatch(tt.args.ctx, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr {
t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}
func Test_postgresqlConnection_CreateTagGroupInBatch(t *testing.T) {
// Setup trow away container
ctx := context.Background()
container, gormDB, err := test.StartPostgresContainer(ctx)
if err != nil {
t.Fatalf("Could not start PostgreSQL container: %v", err)
}
defer container.Terminate(ctx)
// Setup Test
tags := []models.Tag{
{
Name: "JayTheFerret",
Type: models.Artist,
},
{
Name: "SoXX",
Type: models.Character,
},
{
Name: "Dragon",
Type: models.Species,
},
{
Name: "Fennec",
Type: models.Species,
},
}
err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
if err != nil {
t.Fatal(err)
}
tagGroup := []models.TagGroup{
{
Name: "test1",
TagID: tags[0].Name,
},
{
Name: "test2",
TagID: tags[1].Name,
},
{
Name: "test3",
TagID: tags[2].Name,
},
{
Name: "test4",
TagID: tags[3].Name,
},
}
emptyTagGroup := []models.TagGroup{}
// Test
type args struct {
ctx context.Context
tagGroups []models.TagGroup
batchSize int
}
tests := []struct {
name string
args args
wantErr bool
}{
{
name: "Test 1: Valid Tags",
args: args{
ctx: ctx,
tagGroups: tagGroup,
batchSize: 10,
},
wantErr: false,
},
{
name: "Test 2: Empty Tags",
args: args{
ctx: ctx,
tagGroups: emptyTagGroup,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 3: Nil Tags",
args: args{
ctx: ctx,
tagGroups: nil,
batchSize: 10,
},
wantErr: true,
},
{
name: "Test 4: No batchSize",
args: args{
ctx: ctx,
tagGroups: tagGroup,
batchSize: 0,
},
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
p := &postgresqlConnection{
db: gormDB,
debug: true,
}
if err := p.CreateTagGroupInBatch(tt.args.ctx, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr {
t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr)
}
})
}
}

View File

@ -9,6 +9,8 @@ import (
type Tag interface { type Tag interface {
CreateTag(ctx context.Context, tagName models.AnthroveTagName, tagType models.TagType) error CreateTag(ctx context.Context, tagName models.AnthroveTagName, tagType models.TagType) error
CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error
// GetAllTags retrieves all tags. // GetAllTags retrieves all tags.
GetAllTags(ctx context.Context) ([]models.Tag, error) GetAllTags(ctx context.Context) ([]models.Tag, error)

View File

@ -9,6 +9,8 @@ import (
type TagAlias interface { type TagAlias interface {
CreateTagAlias(ctx context.Context, tagAliasName models.AnthroveTagAliasName, tagID models.AnthroveTagID) error CreateTagAlias(ctx context.Context, tagAliasName models.AnthroveTagAliasName, tagID models.AnthroveTagID) error
CreateTagAliasInBatch(ctx context.Context, tagsAliases []models.TagAlias, batchSize int) error
GetAllTagAlias(ctx context.Context) ([]models.TagAlias, error) GetAllTagAlias(ctx context.Context) ([]models.TagAlias, error)
GetAllTagAliasByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagAlias, error) GetAllTagAliasByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagAlias, error)

View File

@ -9,6 +9,8 @@ import (
type TagGroup interface { type TagGroup interface {
CreateTagGroup(ctx context.Context, tagGroupName models.AnthroveTagGroupName, tagID models.AnthroveTagID) error CreateTagGroup(ctx context.Context, tagGroupName models.AnthroveTagGroupName, tagID models.AnthroveTagID) error
CreateTagGroupInBatch(ctx context.Context, tagsGroups []models.TagGroup, batchSize int) error
GetAllTagGroup(ctx context.Context) ([]models.TagGroup, error) GetAllTagGroup(ctx context.Context) ([]models.TagGroup, error)
GetAllTagGroupByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagGroup, error) GetAllTagGroupByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagGroup, error)