Merge pull request 'develop/batch-jobs' (#13) from develop/batch-jobs into main

Reviewed-on: #13

internal/postgres/post.go

@@ -37,6 +37,39 @@ func CreatePost(ctx context.Context, db *gorm.DB, anthrovePost *models.Post) err
 	return nil
 }
 
+func CreatePostInBatch(ctx context.Context, db *gorm.DB, anthrovePost []models.Post, batchSize int) error {
+	if anthrovePost == nil {
+		return &otterError.EntityValidationFailed{Reason: "anthrovePost cannot be nil"}
+	}
+
+	if len(anthrovePost) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "anthrovePost cannot be empty"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).CreateInBatches(anthrovePost, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(anthrovePost),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func GetPostByAnthroveID(ctx context.Context, db *gorm.DB, anthrovePostID models.AnthrovePostID) (*models.Post, error) {
 
 	if anthrovePostID == "" {

internal/postgres/post_test.go

@@ -3,11 +3,11 @@ package postgres
 import (
 	"context"
 	"fmt"
+	_ "github.com/lib/pq"
 	"testing"
 
 	"git.dragse.it/anthrove/otter-space-sdk/v2/pkg/models"
 	"git.dragse.it/anthrove/otter-space-sdk/v2/test"
-	_ "github.com/lib/pq"
 	"gorm.io/gorm"
 )
 
@@ -361,6 +361,93 @@ func TestGetPostBySourceID(t *testing.T) {
 	}
 }
 
+func TestCreatePostInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Tests
+
+	validPosts := []models.Post{
+		{
+			Rating: models.SFW,
+		},
+		{
+			Rating: models.NSFW,
+		},
+		{
+			Rating: models.Questionable,
+		},
+	}
+
+	emptyPost := []models.Post{}
+
+	// Test
+	type args struct {
+		ctx          context.Context
+		db           *gorm.DB
+		anthrovePost []models.Post
+		batchSize    int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Data",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: validPosts,
+				batchSize:    len(validPosts),
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Emtpy Data",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: emptyPost,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Data",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: nil,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: batchSize 0",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: validPosts,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreatePostInBatch(tt.args.ctx, tt.args.db, tt.args.anthrovePost, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreatePostInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func checkPost(got *models.Post, want *models.Post) bool {
 
 	if got == nil && want == nil {

internal/postgres/tag.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 
+	"gorm.io/gorm/clause"
+
 	otterError "git.dragse.it/anthrove/otter-space-sdk/v2/pkg/error"
 	"git.dragse.it/anthrove/otter-space-sdk/v2/pkg/models"
 	log "github.com/sirupsen/logrus"
@@ -40,6 +42,43 @@ func CreateTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName,
 	return nil
 }
 
+func CreateTagInBatchAndUpdate(ctx context.Context, db *gorm.DB, tags []models.Tag, batchSize int) error {
+	if len(tags) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "tags cannot be empty"}
+	}
+
+	if tags == nil {
+		return &otterError.EntityValidationFailed{Reason: "tags cannot be nil"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).
+		Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "name"}},
+			DoUpdates: clause.AssignmentColumns([]string{"tag_type"}),
+		}).CreateInBatches(tags, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(tags),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func DeleteTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName) error {
 
 	if tagName == "" {
@@ -149,7 +188,10 @@ func CreateTagAlias(ctx context.Context, db *gorm.DB, tagAliasName models.Anthro
 		return &otterError.EntityValidationFailed{Reason: otterError.AnthroveTagIDEmpty}
 	}
 
-	result := db.WithContext(ctx).Create(&models.TagAlias{
+	result := db.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "name"}},
+		DoNothing: true,
+	}).Create(&models.TagAlias{
 		Name:  string(tagAliasName),
 		TagID: string(tagID),
 	})
@@ -169,6 +211,42 @@ func CreateTagAlias(ctx context.Context, db *gorm.DB, tagAliasName models.Anthro
 	return nil
 }
 
+func CreateTagAliasInBatch(ctx context.Context, db *gorm.DB, tagAliases []models.TagAlias, batchSize int) error {
+	if len(tagAliases) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"}
+	}
+
+	if tagAliases == nil {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "name"}},
+		DoNothing: true,
+	}).CreateInBatches(tagAliases, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(tagAliases),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func GetAllTagAlias(ctx context.Context, db *gorm.DB) ([]models.TagAlias, error) {
 	var tagAliases []models.TagAlias
 
@@ -263,6 +341,39 @@ func CreateTagGroup(ctx context.Context, db *gorm.DB, tagGroupName models.Anthro
 	return nil
 }
 
+func CreateTagGroupInBatch(ctx context.Context, db *gorm.DB, tagGroups []models.TagGroup, batchSize int) error {
+	if len(tagGroups) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"}
+	}
+
+	if tagGroups == nil {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).CreateInBatches(tagGroups, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(tagGroups),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func GetAllTagGroup(ctx context.Context, db *gorm.DB) ([]models.TagGroup, error) {
 	var tagGroups []models.TagGroup
 

internal/postgres/tag_test.go

@@ -328,7 +328,7 @@ func TestCreateTagAlias(t *testing.T) {
 				tagAliasName: validTagAliasName01,
 				tagID:        validTagID,
 			},
-			wantErr: true,
+			wantErr: false,
 		},
 		{
 			name: "Test 6: Invalide tagID",
@@ -1128,3 +1128,325 @@ func TestGetAllTagByTagType(t *testing.T) {
 		})
 	}
 }
+
+func TestCreateTagInBatchAndUpdate(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	emptyTags := []models.Tag{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		db        *gorm.DB
+		tags      []models.Tag
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      tags,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      emptyTags,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      nil,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.db, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestCreateTagAliasInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagAlias := []models.TagAlias{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagAlias := []models.TagAlias{}
+
+	// Test
+	type args struct {
+		ctx        context.Context
+		db         *gorm.DB
+		tagAliases []models.TagAlias
+		batchSize  int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: tagAlias,
+				batchSize:  10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: emptyTagAlias,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: nil,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: tagAlias,
+				batchSize:  0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreateTagAliasInBatch(tt.args.ctx, tt.args.db, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestCreateTagGroupInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagGroup := []models.TagGroup{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagGroup := []models.TagGroup{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		db        *gorm.DB
+		tagGroups []models.TagGroup
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: tagGroup,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: emptyTagGroup,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: tagGroup,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreateTagGroupInBatch(tt.args.ctx, tt.args.db, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

pkg/database/post.go

@@ -11,6 +11,9 @@ type Post interface {
 	// CreatePost adds a new post to the database.
 	CreatePost(ctx context.Context, anthrovePost *models.Post) error
 
+	// TODO: Everything
+	CreatePostInBatch(ctx context.Context, anthrovePost []models.Post, batchSize int) error
+
 	// GetPostByAnthroveID retrieves a post by its Anthrove ID.
 	GetPostByAnthroveID(ctx context.Context, anthrovePostID models.AnthrovePostID) (*models.Post, error)
 

pkg/database/postgres.go

@@ -92,6 +92,10 @@ func (p *postgresqlConnection) CreatePost(ctx context.Context, anthrovePost *mod
 	return postgres.CreatePost(ctx, p.db, anthrovePost)
 }
 
+func (p *postgresqlConnection) CreatePostInBatch(ctx context.Context, anthrovePost []models.Post, batchSize int) error {
+	return postgres.CreatePostInBatch(ctx, p.db, anthrovePost, batchSize)
+}
+
 func (p *postgresqlConnection) CreatePostWithReferenceToTagAnd(ctx context.Context, anthrovePostID models.AnthrovePostID, anthroveTag *models.Tag) error {
 	return postgres.CreateTagAndReferenceToPost(ctx, p.db, anthrovePostID, anthroveTag)
 }
@@ -214,6 +218,19 @@ func (p *postgresqlConnection) DeleteTag(ctx context.Context, tagName models.Ant
 	return postgres.DeleteTag(ctx, p.db, tagName)
 }
 
+func (p *postgresqlConnection) CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error {
+	return postgres.CreateTagInBatchAndUpdate(ctx, p.db, tags, batchSize)
+}
+
+func (p *postgresqlConnection) CreateTagAliasInBatch(ctx context.Context, tagAliases []models.TagAlias, batchSize int) error {
+	return postgres.CreateTagAliasInBatch(ctx, p.db, tagAliases, batchSize)
+}
+
+func (p *postgresqlConnection) CreateTagGroupInBatch(ctx context.Context, tagGroups []models.TagGroup, batchSize int) error {
+	return postgres.CreateTagGroupInBatch(ctx, p.db, tagGroups, batchSize)
+
+}
+
 // HELPER
 
 func (p *postgresqlConnection) migrateDatabase(dbPool *gorm.DB) error {

pkg/database/postgres_test.go

@@ -2086,15 +2086,6 @@ func Test_postgresqlConnection_CreateTagAlias(t *testing.T) {
 			},
 			wantErr: true,
 		},
-		{
-			name: "Test 5: Duplicate tagID",
-			args: args{
-				ctx:          ctx,
-				tagAliasName: validTagAliasName01,
-				tagID:        validTagID,
-			},
-			wantErr: true,
-		},
 		{
 			name: "Test 6: Invalide tagID",
 			args: args{
@@ -2118,6 +2109,121 @@ func Test_postgresqlConnection_CreateTagAlias(t *testing.T) {
 	}
 }
 
+func Test_postgresqlConnection_CreateTagAliasInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagAlias := []models.TagAlias{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagAlias := []models.TagAlias{}
+
+	// Test
+	type args struct {
+		ctx        context.Context
+		tagAliases []models.TagAlias
+		batchSize  int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:        ctx,
+				tagAliases: tagAlias,
+				batchSize:  10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:        ctx,
+				tagAliases: emptyTagAlias,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:        ctx,
+				tagAliases: nil,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:        ctx,
+				tagAliases: tagAlias,
+				batchSize:  0,
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreateTagAliasInBatch(tt.args.ctx, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func Test_postgresqlConnection_GetAllTagAlias(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -2467,6 +2573,121 @@ func Test_postgresqlConnection_CreateTagGroup(t *testing.T) {
 	}
 }
 
+func Test_postgresqlConnection_CreateTagGroupInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagGroup := []models.TagGroup{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagGroup := []models.TagGroup{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		tagGroups []models.TagGroup
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				tagGroups: tagGroup,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				tagGroups: emptyTagGroup,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				tagGroups: nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				tagGroups: tagGroup,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreateTagGroupInBatch(tt.args.ctx, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func Test_postgresqlConnection_GetAllTagGroup(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -2718,6 +2939,8 @@ func Test_postgresqlConnection_DeleteTagGroup(t *testing.T) {
 	}
 }
 
+//--------------------------
+
 func Test_postgresqlConnection_UpdateUserSourceScrapeTimeInterval(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -3074,6 +3297,8 @@ func Test_postgresqlConnection_UpdateUserSourceValidation(t *testing.T) {
 	}
 }
 
+//--------------------------
+
 func Test_postgresqlConnection_DeleteTag(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -3252,3 +3477,180 @@ func Test_postgresqlConnection_GetAllTagsByTagType(t *testing.T) {
 		})
 	}
 }
+
+func Test_postgresqlConnection_CreateTagInBatchAndUpdate(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	emptyTags := []models.Tag{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		tags      []models.Tag
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				tags:      tags,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				tags:      emptyTags,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				tags:      nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				tags:      nil,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_postgresqlConnection_CreatePostInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Tests
+
+	validPosts := []models.Post{
+		{
+			Rating: models.SFW,
+		},
+		{
+			Rating: models.NSFW,
+		},
+		{
+			Rating: models.Questionable,
+		},
+	}
+
+	emptyPost := []models.Post{}
+
+	// Test
+	type args struct {
+		ctx          context.Context
+		anthrovePost []models.Post
+		batchSize    int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Data",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: validPosts,
+				batchSize:    len(validPosts),
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Emtpy Data",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: emptyPost,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Data",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: nil,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: batchSize 0",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: validPosts,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreatePostInBatch(tt.args.ctx, tt.args.anthrovePost, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreatePostInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

pkg/database/tag.go

@@ -9,6 +9,8 @@ import (
 type Tag interface {
 	CreateTag(ctx context.Context, tagName models.AnthroveTagName, tagType models.TagType) error
 
+	CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error
+
 	// GetAllTags retrieves all tags.
 	GetAllTags(ctx context.Context) ([]models.Tag, error)
 

pkg/database/tagalias.go

@@ -9,6 +9,8 @@ import (
 type TagAlias interface {
 	CreateTagAlias(ctx context.Context, tagAliasName models.AnthroveTagAliasName, tagID models.AnthroveTagID) error
 
+	CreateTagAliasInBatch(ctx context.Context, tagsAliases []models.TagAlias, batchSize int) error
+
 	GetAllTagAlias(ctx context.Context) ([]models.TagAlias, error)
 
 	GetAllTagAliasByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagAlias, error)

pkg/database/taggroup.go

@@ -9,6 +9,8 @@ import (
 type TagGroup interface {
 	CreateTagGroup(ctx context.Context, tagGroupName models.AnthroveTagGroupName, tagID models.AnthroveTagID) error
 
+	CreateTagGroupInBatch(ctx context.Context, tagsGroups []models.TagGroup, batchSize int) error
+
 	GetAllTagGroup(ctx context.Context) ([]models.TagGroup, error)
 
 	GetAllTagGroupByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagGroup, error)

pkg/models/postReference.go

@@ -3,7 +3,7 @@ package models
 type PostReference struct {
 	PostID   string `gorm:"primaryKey"`
 	SourceID string `gorm:"primaryKey"`
-	URL      string `gorm:"not null;unique"`
+	URL      string `gorm:"primaryKey"`
 	PostReferenceConfig
 }