From d9436c584371b30a1b65e12e86cdad907bf2b87e Mon Sep 17 00:00:00 2001 From: David Janowski Date: Wed, 21 Jun 2023 13:29:23 +0200 Subject: [PATCH] using new logging provider --- database/neo4j/user.go | 3 --- e621/favorite.go | 7 ++++- e621/request.go | 2 -- e621/users.go | 5 ++++ main.go | 19 +++++++++++++- services/user.go | 59 ++++++++++++++++++++++++++++++++---------- 6 files changed, 75 insertions(+), 20 deletions(-) diff --git a/database/neo4j/user.go b/database/neo4j/user.go index cb8ec65..dab7a39 100644 --- a/database/neo4j/user.go +++ b/database/neo4j/user.go @@ -4,7 +4,6 @@ import ( "context" "e621_to_neo4j/e621/models" "github.com/neo4j/neo4j-go-driver/v5/neo4j" - "log" ) func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error { @@ -21,7 +20,5 @@ func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user mo if err != nil { return err } - - log.Println("User node created successfully!") return nil } diff --git a/e621/favorite.go b/e621/favorite.go index a3bcf38..7460825 100644 --- a/e621/favorite.go +++ b/e621/favorite.go @@ -3,7 +3,7 @@ package e621 import ( "e621_to_neo4j/e621/models" "fmt" - "log" + log "github.com/sirupsen/logrus" ) // GetFavorites retrieves all favorites from the e621 API. @@ -14,6 +14,11 @@ func (c *Client) GetFavorites(user models.E621User) ([]models.Post, error) { for { URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page) + log.WithFields(log.Fields{ + "id": user.ID, + "fav_page": page, + "uri": URIPath, + }).Debug("Requesting API for favorites") favorite, err := ExecuteGetAPIRequest[models.PostResponseWrapper](c, URIPath) if err != nil { log.Printf(err.Error()) diff --git a/e621/request.go b/e621/request.go index 38e72f9..c43205b 100644 --- a/e621/request.go +++ b/e621/request.go @@ -5,14 +5,12 @@ import ( "encoding/json" "fmt" "io" - "log" "net/http" ) func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) { var err error c.limiter.Wait(context.Background()) - log.Println(URIPath) url := fmt.Sprintf("%s/%s", baseURL, URIPath) req, err := http.NewRequest("GET", url, nil) if err != nil { diff --git a/e621/users.go b/e621/users.go index 8d2f3fe..b431314 100644 --- a/e621/users.go +++ b/e621/users.go @@ -3,11 +3,16 @@ package e621 import ( "e621_to_neo4j/e621/models" "fmt" + log "github.com/sirupsen/logrus" ) // GetUserInfo retrieves the users information from e621 API. func (c *Client) GetUserInfo(username string) (models.E621User, error) { URIPath := fmt.Sprintf("users/%s.json", username) + log.WithFields(log.Fields{ + "username": username, + "uri": URIPath, + }).Debug("Requesting API for user details") user, err := ExecuteGetAPIRequest[models.E621User](c, URIPath) if err != nil { return models.E621User{}, err diff --git a/main.go b/main.go index 9fdd2d4..13a17a6 100644 --- a/main.go +++ b/main.go @@ -7,12 +7,29 @@ import ( neo4j "e621_to_neo4j/database/neo4j" "e621_to_neo4j/e621" "e621_to_neo4j/utils" - "log" + log "github.com/sirupsen/logrus" "net/http" + "os" "strings" ) +func init() { + // Log as JSON instead of the default ASCII formatter. + //log.SetFormatter(&log.JSONFormatter{}) + + // Output to stdout instead of the default stderr + // Can be any io.Writer, see below for File example + log.SetOutput(os.Stdout) + + // Only log the warning severity or above. + log.SetLevel(log.InfoLevel) + + // Logging Method Name + //log.SetReportCaller(true) +} + func main() { + var graphConnection database.GraphConnection ctx := context.Background() diff --git a/services/user.go b/services/user.go index e45c797..eddb211 100644 --- a/services/user.go +++ b/services/user.go @@ -6,7 +6,7 @@ import ( "e621_to_neo4j/e621" "e621_to_neo4j/e621/models" "e621_to_neo4j/utils" - "log" + log "github.com/sirupsen/logrus" "time" ) @@ -19,32 +19,50 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e } if e621User.IsBanned { - log.Printf("User %s is banned from e621!", e621User.Name) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + "bann": e621User.IsBanned, + }).Info("User is Banned") return nil } - log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + }).Info("Processing user") err = graphConnection.UploadUser(ctx, e621User) if err != nil { log.Fatal(err) } - log.Printf("Getting favorites for user %s with id %d", e621User.Name, e621User.ID) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + }).Info("Getting favorites for user") start := time.Now() userFavorites, err := e621Client.GetFavorites(e621User) if err != nil { log.Fatal(err) } - log.Printf("User %s with id %d has %d favorites. Time took to scrape: %v", e621User.Name, e621User.ID, len(userFavorites), time.Since(start)) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + "post_amount": len(userFavorites), + "scrape_time": time.Since(start), + }).Info("Getting favorites for user") startUploadPosts := time.Now() // Uploads all Tags, Posts as Nodes to Neo4j for i, post := range userFavorites { if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists { - log.Printf("No new posts found for user %s with id %d", e621User.Name, e621User.ID) - log.Printf("Last Post ID Found: %d", post.ID) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + "last_post_id": post.ID, + }).Info("No new favorites found") break } else if err != nil { return err @@ -55,7 +73,14 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e if err != nil { return err } - log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start)) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + "post_number": i, + "post_amount": len(userFavorites), + "post_id": post.ID, + "upload_time": time.Since(start), + }).Debug("Uploading post") start := time.Now() err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User) @@ -93,9 +118,20 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e log.Fatal(err) return err } - log.Printf("Making relationship for user %s with id %d, %d for Post: %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start)) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + "post_number": i, + "post_amount": len(userFavorites), + "post_id": post.ID, + "upload_time": time.Since(start), + }).Debug("Making relationship") } - log.Printf("Uploading all posts for user %s took: %v", username, time.Since(startUploadPosts)) + log.WithFields(log.Fields{ + "user": e621User.Name, + "id": e621User.ID, + "upload_time": time.Since(startUploadPosts), + }).Info("Upload to Database finished") return nil } @@ -240,6 +276,3 @@ func uploadArtistTagRelationship(ctx context.Context, graphConnection database.G } return nil } - -//11min für Selloo, simultan mit mutt_jake -//1h58m53 für mutt_jake, mit Selloo