Merge pull request 'Fixed Pagination & modles' (#14) from develop/orchestrator-fixes into main

Reviewed-on: #14
Reviewed-by: Lennard Brinkhaus <lennard.brinkhaus@noreply.localhost>

internal/postgres/post.go

@@ -37,6 +37,39 @@ func CreatePost(ctx context.Context, db *gorm.DB, anthrovePost *models.Post) err
 	return nil
 }
 
+func CreatePostInBatch(ctx context.Context, db *gorm.DB, anthrovePost []models.Post, batchSize int) error {
+	if anthrovePost == nil {
+		return &otterError.EntityValidationFailed{Reason: "anthrovePost cannot be nil"}
+	}
+
+	if len(anthrovePost) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "anthrovePost cannot be empty"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).CreateInBatches(anthrovePost, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(anthrovePost),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func GetPostByAnthroveID(ctx context.Context, db *gorm.DB, anthrovePostID models.AnthrovePostID) (*models.Post, error) {
 
 	if anthrovePostID == "" {

internal/postgres/post_test.go

@@ -3,11 +3,11 @@ package postgres
 import (
 	"context"
 	"fmt"
+	_ "github.com/lib/pq"
 	"testing"
 
 	"git.dragse.it/anthrove/otter-space-sdk/v2/pkg/models"
 	"git.dragse.it/anthrove/otter-space-sdk/v2/test"
-	_ "github.com/lib/pq"
 	"gorm.io/gorm"
 )
 
@@ -361,6 +361,93 @@ func TestGetPostBySourceID(t *testing.T) {
 	}
 }
 
+func TestCreatePostInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Tests
+
+	validPosts := []models.Post{
+		{
+			Rating: models.SFW,
+		},
+		{
+			Rating: models.NSFW,
+		},
+		{
+			Rating: models.Questionable,
+		},
+	}
+
+	emptyPost := []models.Post{}
+
+	// Test
+	type args struct {
+		ctx          context.Context
+		db           *gorm.DB
+		anthrovePost []models.Post
+		batchSize    int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Data",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: validPosts,
+				batchSize:    len(validPosts),
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Emtpy Data",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: emptyPost,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Data",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: nil,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: batchSize 0",
+			args: args{
+				ctx:          ctx,
+				db:           gormDB,
+				anthrovePost: validPosts,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreatePostInBatch(tt.args.ctx, tt.args.db, tt.args.anthrovePost, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreatePostInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func checkPost(got *models.Post, want *models.Post) bool {
 
 	if got == nil && want == nil {

internal/postgres/tag.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"errors"
 
+	"gorm.io/gorm/clause"
+
 	otterError "git.dragse.it/anthrove/otter-space-sdk/v2/pkg/error"
 	"git.dragse.it/anthrove/otter-space-sdk/v2/pkg/models"
 	log "github.com/sirupsen/logrus"
@@ -40,6 +42,43 @@ func CreateTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName,
 	return nil
 }
 
+func CreateTagInBatchAndUpdate(ctx context.Context, db *gorm.DB, tags []models.Tag, batchSize int) error {
+	if len(tags) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "tags cannot be empty"}
+	}
+
+	if tags == nil {
+		return &otterError.EntityValidationFailed{Reason: "tags cannot be nil"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).
+		Clauses(clause.OnConflict{
+			Columns:   []clause.Column{{Name: "name"}},
+			DoUpdates: clause.AssignmentColumns([]string{"tag_type"}),
+		}).CreateInBatches(tags, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(tags),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func DeleteTag(ctx context.Context, db *gorm.DB, tagName models.AnthroveTagName) error {
 
 	if tagName == "" {
@@ -149,7 +188,10 @@ func CreateTagAlias(ctx context.Context, db *gorm.DB, tagAliasName models.Anthro
 		return &otterError.EntityValidationFailed{Reason: otterError.AnthroveTagIDEmpty}
 	}
 
-	result := db.WithContext(ctx).Create(&models.TagAlias{
+	result := db.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "name"}},
+		DoNothing: true,
+	}).Create(&models.TagAlias{
 		Name:  string(tagAliasName),
 		TagID: string(tagID),
 	})
@@ -169,6 +211,42 @@ func CreateTagAlias(ctx context.Context, db *gorm.DB, tagAliasName models.Anthro
 	return nil
 }
 
+func CreateTagAliasInBatch(ctx context.Context, db *gorm.DB, tagAliases []models.TagAlias, batchSize int) error {
+	if len(tagAliases) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"}
+	}
+
+	if tagAliases == nil {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).Clauses(clause.OnConflict{
+		Columns:   []clause.Column{{Name: "name"}},
+		DoNothing: true,
+	}).CreateInBatches(tagAliases, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(tagAliases),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func GetAllTagAlias(ctx context.Context, db *gorm.DB) ([]models.TagAlias, error) {
 	var tagAliases []models.TagAlias
 
@@ -263,6 +341,39 @@ func CreateTagGroup(ctx context.Context, db *gorm.DB, tagGroupName models.Anthro
 	return nil
 }
 
+func CreateTagGroupInBatch(ctx context.Context, db *gorm.DB, tagGroups []models.TagGroup, batchSize int) error {
+	if len(tagGroups) == 0 {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be empty"}
+	}
+
+	if tagGroups == nil {
+		return &otterError.EntityValidationFailed{Reason: "tagAliases cannot be nil"}
+	}
+
+	if batchSize == 0 {
+		return &otterError.EntityValidationFailed{Reason: "batch size cannot be zero"}
+	}
+
+	result := db.WithContext(ctx).CreateInBatches(tagGroups, batchSize)
+	if result.Error != nil {
+		if errors.Is(result.Error, gorm.ErrDuplicatedKey) {
+			return &otterError.EntityAlreadyExists{}
+		}
+		return result.Error
+	}
+
+	if result.RowsAffected == 0 {
+		return &otterError.NoDataWritten{}
+	}
+
+	log.WithFields(log.Fields{
+		"tag_size":   len(tagGroups),
+		"batch_size": batchSize,
+	}).Trace("database: created tag node")
+
+	return nil
+}
+
 func GetAllTagGroup(ctx context.Context, db *gorm.DB) ([]models.TagGroup, error) {
 	var tagGroups []models.TagGroup
 

internal/postgres/tag_test.go

@@ -328,7 +328,7 @@ func TestCreateTagAlias(t *testing.T) {
 				tagAliasName: validTagAliasName01,
 				tagID:        validTagID,
 			},
-			wantErr: true,
+			wantErr: false,
 		},
 		{
 			name: "Test 6: Invalide tagID",
@@ -1128,3 +1128,325 @@ func TestGetAllTagByTagType(t *testing.T) {
 		})
 	}
 }
+
+func TestCreateTagInBatchAndUpdate(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	emptyTags := []models.Tag{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		db        *gorm.DB
+		tags      []models.Tag
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      tags,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      emptyTags,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tags:      nil,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.db, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestCreateTagAliasInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagAlias := []models.TagAlias{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagAlias := []models.TagAlias{}
+
+	// Test
+	type args struct {
+		ctx        context.Context
+		db         *gorm.DB
+		tagAliases []models.TagAlias
+		batchSize  int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: tagAlias,
+				batchSize:  10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: emptyTagAlias,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: nil,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:        ctx,
+				db:         gormDB,
+				tagAliases: tagAlias,
+				batchSize:  0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreateTagAliasInBatch(tt.args.ctx, tt.args.db, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func TestCreateTagGroupInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagGroup := []models.TagGroup{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagGroup := []models.TagGroup{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		db        *gorm.DB
+		tagGroups []models.TagGroup
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: tagGroup,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: emptyTagGroup,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				db:        gormDB,
+				tagGroups: tagGroup,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if err := CreateTagGroupInBatch(tt.args.ctx, tt.args.db, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}

internal/postgres/user.go

@@ -230,8 +230,8 @@ func GetAllUsers(ctx context.Context, db *gorm.DB) ([]models.User, error) {
 	return users, nil
 }
 
+// TODO: FIX THE TEST
 func GetUserFavoriteWithPagination(ctx context.Context, db *gorm.DB, anthroveUserID models.AnthroveUserID, skip int, limit int) (*models.FavoriteList, error) {
-	var userFavorites []models.UserFavorites
 	var favoritePosts []models.Post
 
 	if anthroveUserID == "" {
@@ -242,30 +242,7 @@ func GetUserFavoriteWithPagination(ctx context.Context, db *gorm.DB, anthroveUse
 		return nil, &otterError.EntityValidationFailed{Reason: otterError.AnthroveUserIDToShort}
 	}
 
-	err := db.WithContext(ctx).Model(&models.UserFavorites{}).Where("user_id = ?", string(anthroveUserID)).Offset(skip).Limit(limit).Find(&userFavorites).Error
+	db.WithContext(ctx).Joins("RIGHT JOIN \"UserFavorites\" AS of ON \"Post\".id = of.post_id AND of.user_id = ?", anthroveUserID).Preload("References").Offset(skip).Limit(limit).Find(&favoritePosts)
-	if err != nil {
-		if errors.Is(err, gorm.ErrRecordNotFound) {
-			return nil, &otterError.NoDataFound{}
-		}
-		return nil, err
-	}
-
-	for _, userFavorite := range userFavorites {
-		var post models.Post
-		err = db.WithContext(ctx).Model(&models.Post{}).Where("id = ?", userFavorite.PostID).First(&post).Error
-		if err != nil {
-			if errors.Is(err, gorm.ErrRecordNotFound) {
-				return nil, &otterError.NoDataFound{}
-			}
-			return nil, err
-		}
-
-		favoritePosts = append(favoritePosts,
-			models.Post{
-				BaseModel: models.BaseModel[models.AnthrovePostID]{ID: post.ID},
-				Rating:    post.Rating,
-			})
-	}
 
 	log.WithFields(log.Fields{
 		"anthrove_user_id":        anthroveUserID,

internal/postgres/user_test.go

@@ -573,13 +573,30 @@ func TestGetUserFavoriteNodeWithPagination(t *testing.T) {
 				t.Errorf("GetAllUserFavoritesWithPagination() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(got, tt.want) {
+			if !checkFavoritePosts(got, tt.want) {
 				t.Errorf("GetAllUserFavoritesWithPagination() got = %v, want %v", got, tt.want)
 			}
 		})
 	}
 }
 
+func checkFavoritePosts(got *models.FavoriteList, want *models.FavoriteList) bool {
+	if got == nil && want == nil {
+		return true
+	} else if got == nil || want == nil {
+		return false
+	}
+
+	for i, post := range got.Posts {
+		if post.ID == want.Posts[i].ID {
+		} else {
+			return false
+		}
+	}
+
+	return true
+}
+
 func TestGetUserFavoritesCount(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()

pkg/database/post.go

@@ -11,6 +11,9 @@ type Post interface {
 	// CreatePost adds a new post to the database.
 	CreatePost(ctx context.Context, anthrovePost *models.Post) error
 
+	// TODO: Everything
+	CreatePostInBatch(ctx context.Context, anthrovePost []models.Post, batchSize int) error
+
 	// GetPostByAnthroveID retrieves a post by its Anthrove ID.
 	GetPostByAnthroveID(ctx context.Context, anthrovePostID models.AnthrovePostID) (*models.Post, error)
 

pkg/database/postgres.go

@@ -92,6 +92,10 @@ func (p *postgresqlConnection) CreatePost(ctx context.Context, anthrovePost *mod
 	return postgres.CreatePost(ctx, p.db, anthrovePost)
 }
 
+func (p *postgresqlConnection) CreatePostInBatch(ctx context.Context, anthrovePost []models.Post, batchSize int) error {
+	return postgres.CreatePostInBatch(ctx, p.db, anthrovePost, batchSize)
+}
+
 func (p *postgresqlConnection) CreatePostWithReferenceToTagAnd(ctx context.Context, anthrovePostID models.AnthrovePostID, anthroveTag *models.Tag) error {
 	return postgres.CreateTagAndReferenceToPost(ctx, p.db, anthrovePostID, anthroveTag)
 }
@@ -214,6 +218,19 @@ func (p *postgresqlConnection) DeleteTag(ctx context.Context, tagName models.Ant
 	return postgres.DeleteTag(ctx, p.db, tagName)
 }
 
+func (p *postgresqlConnection) CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error {
+	return postgres.CreateTagInBatchAndUpdate(ctx, p.db, tags, batchSize)
+}
+
+func (p *postgresqlConnection) CreateTagAliasInBatch(ctx context.Context, tagAliases []models.TagAlias, batchSize int) error {
+	return postgres.CreateTagAliasInBatch(ctx, p.db, tagAliases, batchSize)
+}
+
+func (p *postgresqlConnection) CreateTagGroupInBatch(ctx context.Context, tagGroups []models.TagGroup, batchSize int) error {
+	return postgres.CreateTagGroupInBatch(ctx, p.db, tagGroups, batchSize)
+
+}
+
 // HELPER
 
 func (p *postgresqlConnection) migrateDatabase(dbPool *gorm.DB) error {

pkg/database/postgres_test.go

@@ -1556,7 +1556,7 @@ func Test_postgresqlConnection_GetUserFavoriteWithPagination(t *testing.T) {
 				t.Errorf("GetAllUserFavoritesWithPagination() error = %v, wantErr %v", err, tt.wantErr)
 				return
 			}
-			if !reflect.DeepEqual(got, tt.want) {
+			if !checkFavoritePosts(got, tt.want) {
 				t.Errorf("GetAllUserFavoritesWithPagination() got = %v, want %v", got, tt.want)
 			}
 		})
@@ -2086,15 +2086,6 @@ func Test_postgresqlConnection_CreateTagAlias(t *testing.T) {
 			},
 			wantErr: true,
 		},
-		{
-			name: "Test 5: Duplicate tagID",
-			args: args{
-				ctx:          ctx,
-				tagAliasName: validTagAliasName01,
-				tagID:        validTagID,
-			},
-			wantErr: true,
-		},
 		{
 			name: "Test 6: Invalide tagID",
 			args: args{
@@ -2118,6 +2109,121 @@ func Test_postgresqlConnection_CreateTagAlias(t *testing.T) {
 	}
 }
 
+func Test_postgresqlConnection_CreateTagAliasInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagAlias := []models.TagAlias{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagAlias := []models.TagAlias{}
+
+	// Test
+	type args struct {
+		ctx        context.Context
+		tagAliases []models.TagAlias
+		batchSize  int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:        ctx,
+				tagAliases: tagAlias,
+				batchSize:  10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:        ctx,
+				tagAliases: emptyTagAlias,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:        ctx,
+				tagAliases: nil,
+				batchSize:  10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:        ctx,
+				tagAliases: tagAlias,
+				batchSize:  0,
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreateTagAliasInBatch(tt.args.ctx, tt.args.tagAliases, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagAliasInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func Test_postgresqlConnection_GetAllTagAlias(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -2467,6 +2573,121 @@ func Test_postgresqlConnection_CreateTagGroup(t *testing.T) {
 	}
 }
 
+func Test_postgresqlConnection_CreateTagGroupInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	err = postgres.CreateTagInBatchAndUpdate(ctx, gormDB, tags, len(tags))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	tagGroup := []models.TagGroup{
+		{
+			Name:  "test1",
+			TagID: tags[0].Name,
+		},
+		{
+			Name:  "test2",
+			TagID: tags[1].Name,
+		},
+		{
+			Name:  "test3",
+			TagID: tags[2].Name,
+		},
+		{
+			Name:  "test4",
+			TagID: tags[3].Name,
+		},
+	}
+	emptyTagGroup := []models.TagGroup{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		tagGroups []models.TagGroup
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				tagGroups: tagGroup,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				tagGroups: emptyTagGroup,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				tagGroups: nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				tagGroups: tagGroup,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreateTagGroupInBatch(tt.args.ctx, tt.args.tagGroups, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagGroupInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
 func Test_postgresqlConnection_GetAllTagGroup(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -2718,6 +2939,8 @@ func Test_postgresqlConnection_DeleteTagGroup(t *testing.T) {
 	}
 }
 
+//--------------------------
+
 func Test_postgresqlConnection_UpdateUserSourceScrapeTimeInterval(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -3074,6 +3297,8 @@ func Test_postgresqlConnection_UpdateUserSourceValidation(t *testing.T) {
 	}
 }
 
+//--------------------------
+
 func Test_postgresqlConnection_DeleteTag(t *testing.T) {
 	// Setup trow away container
 	ctx := context.Background()
@@ -3252,3 +3477,197 @@ func Test_postgresqlConnection_GetAllTagsByTagType(t *testing.T) {
 		})
 	}
 }
+
+func Test_postgresqlConnection_CreateTagInBatchAndUpdate(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Test
+	tags := []models.Tag{
+		{
+			Name: "JayTheFerret",
+			Type: models.Artist,
+		},
+		{
+			Name: "SoXX",
+			Type: models.Character,
+		},
+		{
+			Name: "Dragon",
+			Type: models.Species,
+		},
+		{
+			Name: "Fennec",
+			Type: models.Species,
+		},
+	}
+	emptyTags := []models.Tag{}
+
+	// Test
+	type args struct {
+		ctx       context.Context
+		tags      []models.Tag
+		batchSize int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Tags",
+			args: args{
+				ctx:       ctx,
+				tags:      tags,
+				batchSize: 10,
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Empty Tags",
+			args: args{
+				ctx:       ctx,
+				tags:      emptyTags,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Tags",
+			args: args{
+				ctx:       ctx,
+				tags:      nil,
+				batchSize: 10,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: No batchSize",
+			args: args{
+				ctx:       ctx,
+				tags:      nil,
+				batchSize: 0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreateTagInBatchAndUpdate(tt.args.ctx, tt.args.tags, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreateTagInBatchAndUpdate() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func Test_postgresqlConnection_CreatePostInBatch(t *testing.T) {
+	// Setup trow away container
+	ctx := context.Background()
+	container, gormDB, err := test.StartPostgresContainer(ctx)
+	if err != nil {
+		t.Fatalf("Could not start PostgreSQL container: %v", err)
+	}
+	defer container.Terminate(ctx)
+
+	// Setup Tests
+
+	validPosts := []models.Post{
+		{
+			Rating: models.SFW,
+		},
+		{
+			Rating: models.NSFW,
+		},
+		{
+			Rating: models.Questionable,
+		},
+	}
+
+	emptyPost := []models.Post{}
+
+	// Test
+	type args struct {
+		ctx          context.Context
+		anthrovePost []models.Post
+		batchSize    int
+	}
+	tests := []struct {
+		name    string
+		args    args
+		wantErr bool
+	}{
+		{
+			name: "Test 1: Valid Data",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: validPosts,
+				batchSize:    len(validPosts),
+			},
+			wantErr: false,
+		},
+		{
+			name: "Test 2: Emtpy Data",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: emptyPost,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 3: Nil Data",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: nil,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+		{
+			name: "Test 4: batchSize 0",
+			args: args{
+				ctx:          ctx,
+				anthrovePost: validPosts,
+				batchSize:    0,
+			},
+			wantErr: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			p := &postgresqlConnection{
+				db:    gormDB,
+				debug: true,
+			}
+			if err := p.CreatePostInBatch(tt.args.ctx, tt.args.anthrovePost, tt.args.batchSize); (err != nil) != tt.wantErr {
+				t.Errorf("CreatePostInBatch() error = %v, wantErr %v", err, tt.wantErr)
+			}
+		})
+	}
+}
+
+func checkFavoritePosts(got *models.FavoriteList, want *models.FavoriteList) bool {
+	if got == nil && want == nil {
+		return true
+	} else if got == nil || want == nil {
+		return false
+	}
+
+	for i, post := range got.Posts {
+		if post.ID == want.Posts[i].ID {
+		} else {
+			return false
+		}
+	}
+
+	return true
+}

pkg/database/tag.go

@@ -9,6 +9,8 @@ import (
 type Tag interface {
 	CreateTag(ctx context.Context, tagName models.AnthroveTagName, tagType models.TagType) error
 
+	CreateTagInBatchAndUpdate(ctx context.Context, tags []models.Tag, batchSize int) error
+
 	// GetAllTags retrieves all tags.
 	GetAllTags(ctx context.Context) ([]models.Tag, error)
 

pkg/database/tagalias.go

@@ -9,6 +9,8 @@ import (
 type TagAlias interface {
 	CreateTagAlias(ctx context.Context, tagAliasName models.AnthroveTagAliasName, tagID models.AnthroveTagID) error
 
+	CreateTagAliasInBatch(ctx context.Context, tagsAliases []models.TagAlias, batchSize int) error
+
 	GetAllTagAlias(ctx context.Context) ([]models.TagAlias, error)
 
 	GetAllTagAliasByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagAlias, error)

pkg/database/taggroup.go

@@ -9,6 +9,8 @@ import (
 type TagGroup interface {
 	CreateTagGroup(ctx context.Context, tagGroupName models.AnthroveTagGroupName, tagID models.AnthroveTagID) error
 
+	CreateTagGroupInBatch(ctx context.Context, tagsGroups []models.TagGroup, batchSize int) error
+
 	GetAllTagGroup(ctx context.Context) ([]models.TagGroup, error)
 
 	GetAllTagGroupByTag(ctx context.Context, tagID models.AnthroveTagID) ([]models.TagGroup, error)

pkg/models/orm.go

@@ -1,9 +1,10 @@
 package models
 
 import (
+	"time"
+
 	gonanoid "github.com/matoous/go-nanoid/v2"
 	"gorm.io/gorm"
-	"time"
 )
 
 type ID interface {
@@ -11,10 +12,10 @@ type ID interface {
 }
 
 type BaseModel[T ID] struct {
-	ID        T `gorm:"primaryKey"`
+	ID        T              `json:"id" gorm:"primaryKey"`
-	CreatedAt time.Time
+	CreatedAt time.Time      `json:"-"`
-	UpdatedAt time.Time
+	UpdatedAt time.Time      `json:"-"`
-	DeletedAt gorm.DeletedAt `gorm:"index"`
+	DeletedAt gorm.DeletedAt `json:"-" gorm:"index"`
 }
 
 func (base *BaseModel[T]) BeforeCreate(db *gorm.DB) error {

pkg/models/post.go

@@ -3,10 +3,10 @@ package models
 // Post model
 type Post struct {
 	BaseModel[AnthrovePostID]
-	Rating     Rating          `gorm:"type:enum('safe','questionable','explicit')"`
+	Rating     Rating          `json:"rating" gorm:"type:enum('safe','questionable','explicit')"`
-	Tags       []Tag           `gorm:"many2many:post_tags;"`
+	Tags       []Tag           `json:"-" gorm:"many2many:post_tags;"`
-	Favorites  []UserFavorites `gorm:"foreignKey:PostID"`
+	Favorites  []UserFavorites `json:"-" gorm:"foreignKey:PostID"`
-	References []PostReference `gorm:"foreignKey:PostID"`
+	References []PostReference `json:"references" gorm:"foreignKey:PostID"`
 }
 
 func (Post) TableName() string {

pkg/models/postReference.go

@@ -1,17 +1,17 @@
 package models
 
 type PostReference struct {
-	PostID   string `gorm:"primaryKey"`
+	PostID   string `json:"post_id" gorm:"primaryKey"`
-	SourceID string `gorm:"primaryKey"`
+	SourceID string `json:"source_id" gorm:"primaryKey"`
-	URL      string `gorm:"not null;unique"`
+	URL      string `json:"url" gorm:"primaryKey"`
 	PostReferenceConfig
 }
 
 type PostReferenceConfig struct {
-	SourcePostID   string
+	SourcePostID   string `json:"source_post_id"`
-	FullFileURL    string
+	FullFileURL    string `json:"full_file_url"`
-	PreviewFileURL string
+	PreviewFileURL string `json:"preview_file_url"`
-	SampleFileURL  string
+	SampleFileURL  string `json:"sample_file_url"`
 }
 
 func (PostReference) TableName() string {

pkg/models/source.go

@@ -3,11 +3,11 @@ package models
 // Source model
 type Source struct {
 	BaseModel[AnthroveSourceID]
-	DisplayName string
+	DisplayName string          `json:"display_name" `
-	Domain      string          `gorm:"not null;unique"`
+	Domain      string          `json:"domain" gorm:"not null;unique"`
-	Icon        string          `gorm:"not null"`
+	Icon        string          `json:"icon" gorm:"not null"`
-	UserSources []UserSource    `gorm:"foreignKey:SourceID"`
+	UserSources []UserSource    `json:"-" gorm:"foreignKey:SourceID"`
-	References  []PostReference `gorm:"foreignKey:SourceID"`
+	References  []PostReference `json:"references" gorm:"foreignKey:SourceID"`
 }
 
 func (Source) TableName() string {

pkg/models/tag.go

@@ -2,11 +2,11 @@ package models
 
 // Tag models
 type Tag struct {
-	Name    string     `gorm:"primaryKey"`
+	Name    string     `json:"name" gorm:"primaryKey"`
-	Type    TagType    `gorm:"column:tag_type"`
+	Type    TagType    `json:"type" gorm:"column:tag_type"`
-	Aliases []TagAlias `gorm:"foreignKey:TagID"`
+	Aliases []TagAlias `json:"aliases" gorm:"foreignKey:TagID"`
-	Groups  []TagGroup `gorm:"foreignKey:TagID"`
+	Groups  []TagGroup `json:"groups" gorm:"foreignKey:TagID"`
-	Posts   []Post     `gorm:"many2many:post_tags;"`
+	Posts   []Post     `json:"posts" gorm:"many2many:post_tags;"`
 }
 
 func (Tag) TableName() string {
@@ -15,8 +15,8 @@ func (Tag) TableName() string {
 
 // TagAlias model
 type TagAlias struct {
-	Name  string `gorm:"primaryKey"`
+	Name  string `json:"name" gorm:"primaryKey"`
-	TagID string
+	TagID string `json:"tag_id"`
 }
 
 func (TagAlias) TableName() string {
@@ -25,8 +25,8 @@ func (TagAlias) TableName() string {
 
 // TagGroup model
 type TagGroup struct {
-	Name  string `gorm:"primaryKey"`
+	Name  string `json:"name" gorm:"primaryKey"`
-	TagID string
+	TagID string `json:"tag_id"`
 }
 
 func (TagGroup) TableName() string {

pkg/models/user.go

@@ -3,8 +3,8 @@ package models
 // User model
 type User struct {
 	BaseModel[AnthroveUserID]
-	Favorites []UserFavorites `gorm:"foreignKey:UserID"`
+	Favorites []UserFavorites `json:"-" gorm:"foreignKey:UserID"`
-	Sources   []UserSource    `gorm:"foreignKey:UserID"`
+	Sources   []UserSource    `json:"-" gorm:"foreignKey:UserID"`
 }
 
 func (User) TableName() string {

pkg/models/userFavorite.go

@@ -3,9 +3,9 @@ package models
 import "time"
 
 type UserFavorites struct {
-	UserID    string `gorm:"primaryKey"`
+	UserID    string    `json:"user_id" gorm:"primaryKey"`
-	PostID    string `gorm:"primaryKey"`
+	PostID    string    `json:"post_id" gorm:"primaryKey"`
-	CreatedAt time.Time
+	CreatedAt time.Time `json:"-"`
 }
 
 func (UserFavorites) TableName() string {

pkg/models/userSource.go

@@ -3,16 +3,16 @@ package models
 import "time"
 
 type UserSource struct {
-	User                 User   `gorm:"foreignKey:ID;references:UserID"`
+	User                 User      `json:"user" gorm:"foreignKey:ID;references:UserID"`
-	UserID               string `gorm:"primaryKey"`
+	UserID               string    `json:"user_id" gorm:"primaryKey"`
-	Source               Source `gorm:"foreignKey:ID;references:SourceID"`
+	Source               Source    `json:"source" gorm:"foreignKey:ID;references:SourceID"`
-	SourceID             string `gorm:"primaryKey"`
+	SourceID             string    `json:"source_id" gorm:"primaryKey"`
-	ScrapeTimeInterval   string
+	ScrapeTimeInterval   string    `json:"scrape_time_interval"`
-	AccountUsername      string
+	AccountUsername      string    `json:"account_username"`
-	AccountID            string
+	AccountID            string    `json:"account_id"`
-	LastScrapeTime       time.Time
+	LastScrapeTime       time.Time `json:"last_scrape_time"`
-	AccountValidate      bool
+	AccountValidate      bool      `json:"account_validate"`
-	AccountValidationKey string
+	AccountValidationKey string    `json:"-"`
 }
 
 func (UserSource) TableName() string {