package service import ( "context" "git.dragse.it/anthrove/e621-to-graph/internal/e621" "git.dragse.it/anthrove/e621-to-graph/pkg/e621/model" "git.dragse.it/anthrove/e621-to-graph/pkg/logic" "git.dragse.it/anthrove/e621-to-graph/pkg/util" log "github.com/sirupsen/logrus" "time" ) func ScrapeUser(ctx context.Context, graphConnection logic.GraphConnection, e621Client e621.Client, username string) error { var err error var page int64 = 1 var allFavorites []model.Post AwaitE621User := e621Client.GetUserInfo(username) e621User, err := AwaitE621User() if err != nil { log.Info(err) return err } if e621User.IsBanned { log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, "bann": e621User.IsBanned, }).Info("User is Banned") return nil } log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, }).Info("Processing user") err = graphConnection.UploadUser(ctx, e621User) if err != nil { log.Fatal(err) } log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, }).Info("Getting favorites for user") start := time.Now() for { AwaitE621Favorites := e621Client.GetFavorites(ctx, e621User, page) log.WithFields(log.Fields{ "id": e621User.ID, "fav_page": page, }).Debug("Requesting API for favorites") favoritesPage, err := AwaitE621Favorites() if err != nil { log.Printf(err.Error()) } // Append the fetched posts to the result slice allFavorites = append(allFavorites, favoritesPage.Posts...) // If no more posts are returned, return the accumulated favorites if len(favoritesPage.Posts) == 0 { break } // Update the last post ID for the next page request page += 1 } if err != nil { log.Fatal(err) } log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, "post_amount": len(allFavorites), "scrape_time": time.Since(start), }).Info("Getting favorites for user") startUploadPosts := time.Now() // Uploads all Tags, Posts as Nodes to Neo4j for i, post := range allFavorites { if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists { log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, "last_post_id": post.ID, }).Info("No new favorites found") break } else if err != nil { return err } start = time.Now() err = uploadNodes(ctx, graphConnection, post) if err != nil { return err } log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, "post_number": i, "post_amount": len(allFavorites), "post_id": post.ID, "upload_time": time.Since(start), }).Debug("Uploading post") start := time.Now() err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User) if err != nil { log.Fatal(err) return err } err = uploadSourceTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } err = uploadGeneralTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } err = uploadCharacterTagtRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } err = uploadCopyrightTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } err = uploadArtistTagRelationship(ctx, graphConnection, post) if err != nil { log.Fatal(err) return err } log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, "post_number": i, "post_amount": len(allFavorites), "post_id": post.ID, "upload_time": time.Since(start), }).Debug("Making relationship") } log.WithFields(log.Fields{ "user": e621User.Name, "id": e621User.ID, "upload_time": time.Since(startUploadPosts), }).Info("Upload to Database finished") return nil } // uploadNodes uploads the post to the database and creates the nodes func uploadNodes(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { uniqueGeneralTags := make([]string, 0) uniqueCharacterTags := make([]string, 0) uniqueCopyrightTags := make([]string, 0) uniqueArtistTags := make([]string, 0) allGeneralTags := make([]string, 0) allCharacterTags := make([]string, 0) allCopyrightTags := make([]string, 0) allArtistTags := make([]string, 0) allGeneralTags = append(allGeneralTags, post.Tags.General...) allCharacterTags = append(allCharacterTags, post.Tags.Character...) allCopyrightTags = append(allCopyrightTags, post.Tags.Copyright...) allArtistTags = append(allArtistTags, post.Tags.Artist...) uniqueGeneralTags = util.UniqueNonEmptyElementsOf(allGeneralTags) uniqueCharacterTags = util.UniqueNonEmptyElementsOf(allCharacterTags) uniqueCopyrightTags = util.UniqueNonEmptyElementsOf(allCopyrightTags) uniqueArtistTags = util.UniqueNonEmptyElementsOf(allArtistTags) err := graphConnection.UploadPost(ctx, post.ID) if err != nil { return err } // Uploads the source to the database for _, source := range post.Sources { err := graphConnection.UploadSource(ctx, source) if err != nil { return err } } for _, uniqueGeneralTag := range uniqueGeneralTags { err := graphConnection.UploadTag(ctx, uniqueGeneralTag, "general") if err != nil { return err } } for _, uniqueCharacterTag := range uniqueCharacterTags { err := graphConnection.UploadTag(ctx, uniqueCharacterTag, "character") if err != nil { return err } } for _, uniqueCopyrightTag := range uniqueCopyrightTags { err := graphConnection.UploadTag(ctx, uniqueCopyrightTag, "copyright") if err != nil { return err } } for _, uniqueArtistTag := range uniqueArtistTags { err := graphConnection.UploadTag(ctx, uniqueArtistTag, "artist") if err != nil { return err } } return nil } // uploadPostToUserRelationship creates a relationship between the user and the post func uploadPostToUserRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post, e621User model.E621User) error { err := graphConnection.EstablishUserToPostLink(ctx, post.ID, e621User.ID) if err != nil { return err } // log.Printf("Created UserToPostRelationship for user: %s to post: %d", e621User.Name, post.ID) return nil } // uploadSourceTagRelationship creates a relationship between the post and the source func uploadSourceTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, source := range post.Sources { err := graphConnection.EstablishPostToSourceLink(ctx, post.ID, source) if err != nil { return err } // log.Printf("Created PostToSourceRelationship for Post: %d to source: %s", post.ID, source) } return nil } // uploadGeneralTagRelationship creates a relationship between the post and the general tag func uploadGeneralTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, generalTag := range post.Tags.General { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, generalTag) if err != nil { return err } // log.Printf("Created PostToTagRelationship for post: %d to general tag: %s", post.ID, generalTag) } return nil } // uploadCharacterTagtRelationship creates a relationship between the post and the character tag func uploadCharacterTagtRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, characterTag := range post.Tags.Character { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, characterTag) if err != nil { return err } // log.Printf("Created PostToTagRelationship for post: %d to character tag: %s", post.ID, characterTag) } return nil } // uploadCopyrightTagRelationship creates a relationship between the post and the copyright tag func uploadCopyrightTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, copyrightTag := range post.Tags.Copyright { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, copyrightTag) if err != nil { return err } // log.Printf("Created PostToTagRelationship for post: %d to copyright tag: %s", post.ID, copyrightTag) } return nil } // uploadArtistTagRelationship creates a relationship between the post and the artist tag func uploadArtistTagRelationship(ctx context.Context, graphConnection logic.GraphConnection, post model.Post) error { for _, artistTag := range post.Tags.Artist { err := graphConnection.EstablishPostToTagLink(ctx, post.ID, artistTag) if err != nil { return err } // log.Printf("Created PostToTagRelationship for post: %d to artist tag: %s", post.ID, artistTag) } return nil }