using new logging provider

This commit is contained in:
David Janowski 2023-06-21 13:29:23 +02:00
parent f95e9b0853
commit d9436c5843
6 changed files with 75 additions and 20 deletions

View File

@ -4,7 +4,6 @@ import (
"context" "context"
"e621_to_neo4j/e621/models" "e621_to_neo4j/e621/models"
"github.com/neo4j/neo4j-go-driver/v5/neo4j" "github.com/neo4j/neo4j-go-driver/v5/neo4j"
"log"
) )
func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error { func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error {
@ -21,7 +20,5 @@ func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user mo
if err != nil { if err != nil {
return err return err
} }
log.Println("User node created successfully!")
return nil return nil
} }

View File

@ -3,7 +3,7 @@ package e621
import ( import (
"e621_to_neo4j/e621/models" "e621_to_neo4j/e621/models"
"fmt" "fmt"
"log" log "github.com/sirupsen/logrus"
) )
// GetFavorites retrieves all favorites from the e621 API. // GetFavorites retrieves all favorites from the e621 API.
@ -14,6 +14,11 @@ func (c *Client) GetFavorites(user models.E621User) ([]models.Post, error) {
for { for {
URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page) URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page)
log.WithFields(log.Fields{
"id": user.ID,
"fav_page": page,
"uri": URIPath,
}).Debug("Requesting API for favorites")
favorite, err := ExecuteGetAPIRequest[models.PostResponseWrapper](c, URIPath) favorite, err := ExecuteGetAPIRequest[models.PostResponseWrapper](c, URIPath)
if err != nil { if err != nil {
log.Printf(err.Error()) log.Printf(err.Error())

View File

@ -5,14 +5,12 @@ import (
"encoding/json" "encoding/json"
"fmt" "fmt"
"io" "io"
"log"
"net/http" "net/http"
) )
func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) { func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) {
var err error var err error
c.limiter.Wait(context.Background()) c.limiter.Wait(context.Background())
log.Println(URIPath)
url := fmt.Sprintf("%s/%s", baseURL, URIPath) url := fmt.Sprintf("%s/%s", baseURL, URIPath)
req, err := http.NewRequest("GET", url, nil) req, err := http.NewRequest("GET", url, nil)
if err != nil { if err != nil {

View File

@ -3,11 +3,16 @@ package e621
import ( import (
"e621_to_neo4j/e621/models" "e621_to_neo4j/e621/models"
"fmt" "fmt"
log "github.com/sirupsen/logrus"
) )
// GetUserInfo retrieves the users information from e621 API. // GetUserInfo retrieves the users information from e621 API.
func (c *Client) GetUserInfo(username string) (models.E621User, error) { func (c *Client) GetUserInfo(username string) (models.E621User, error) {
URIPath := fmt.Sprintf("users/%s.json", username) URIPath := fmt.Sprintf("users/%s.json", username)
log.WithFields(log.Fields{
"username": username,
"uri": URIPath,
}).Debug("Requesting API for user details")
user, err := ExecuteGetAPIRequest[models.E621User](c, URIPath) user, err := ExecuteGetAPIRequest[models.E621User](c, URIPath)
if err != nil { if err != nil {
return models.E621User{}, err return models.E621User{}, err

19
main.go
View File

@ -7,12 +7,29 @@ import (
neo4j "e621_to_neo4j/database/neo4j" neo4j "e621_to_neo4j/database/neo4j"
"e621_to_neo4j/e621" "e621_to_neo4j/e621"
"e621_to_neo4j/utils" "e621_to_neo4j/utils"
"log" log "github.com/sirupsen/logrus"
"net/http" "net/http"
"os"
"strings" "strings"
) )
func init() {
// Log as JSON instead of the default ASCII formatter.
//log.SetFormatter(&log.JSONFormatter{})
// Output to stdout instead of the default stderr
// Can be any io.Writer, see below for File example
log.SetOutput(os.Stdout)
// Only log the warning severity or above.
log.SetLevel(log.InfoLevel)
// Logging Method Name
//log.SetReportCaller(true)
}
func main() { func main() {
var graphConnection database.GraphConnection var graphConnection database.GraphConnection
ctx := context.Background() ctx := context.Background()

View File

@ -6,7 +6,7 @@ import (
"e621_to_neo4j/e621" "e621_to_neo4j/e621"
"e621_to_neo4j/e621/models" "e621_to_neo4j/e621/models"
"e621_to_neo4j/utils" "e621_to_neo4j/utils"
"log" log "github.com/sirupsen/logrus"
"time" "time"
) )
@ -19,32 +19,50 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
} }
if e621User.IsBanned { if e621User.IsBanned {
log.Printf("User %s is banned from e621!", e621User.Name) log.WithFields(log.Fields{
"user": e621User.Name,
"id": e621User.ID,
"bann": e621User.IsBanned,
}).Info("User is Banned")
return nil return nil
} }
log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID) log.WithFields(log.Fields{
"user": e621User.Name,
"id": e621User.ID,
}).Info("Processing user")
err = graphConnection.UploadUser(ctx, e621User) err = graphConnection.UploadUser(ctx, e621User)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
log.Printf("Getting favorites for user %s with id %d", e621User.Name, e621User.ID) log.WithFields(log.Fields{
"user": e621User.Name,
"id": e621User.ID,
}).Info("Getting favorites for user")
start := time.Now() start := time.Now()
userFavorites, err := e621Client.GetFavorites(e621User) userFavorites, err := e621Client.GetFavorites(e621User)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
log.Printf("User %s with id %d has %d favorites. Time took to scrape: %v", e621User.Name, e621User.ID, len(userFavorites), time.Since(start)) log.WithFields(log.Fields{
"user": e621User.Name,
"id": e621User.ID,
"post_amount": len(userFavorites),
"scrape_time": time.Since(start),
}).Info("Getting favorites for user")
startUploadPosts := time.Now() startUploadPosts := time.Now()
// Uploads all Tags, Posts as Nodes to Neo4j // Uploads all Tags, Posts as Nodes to Neo4j
for i, post := range userFavorites { for i, post := range userFavorites {
if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists { if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists {
log.Printf("No new posts found for user %s with id %d", e621User.Name, e621User.ID) log.WithFields(log.Fields{
log.Printf("Last Post ID Found: %d", post.ID) "user": e621User.Name,
"id": e621User.ID,
"last_post_id": post.ID,
}).Info("No new favorites found")
break break
} else if err != nil { } else if err != nil {
return err return err
@ -55,7 +73,14 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
if err != nil { if err != nil {
return err return err
} }
log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start)) log.WithFields(log.Fields{
"user": e621User.Name,
"id": e621User.ID,
"post_number": i,
"post_amount": len(userFavorites),
"post_id": post.ID,
"upload_time": time.Since(start),
}).Debug("Uploading post")
start := time.Now() start := time.Now()
err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User) err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User)
@ -93,9 +118,20 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
log.Fatal(err) log.Fatal(err)
return err return err
} }
log.Printf("Making relationship for user %s with id %d, %d for Post: %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start)) log.WithFields(log.Fields{
"user": e621User.Name,
"id": e621User.ID,
"post_number": i,
"post_amount": len(userFavorites),
"post_id": post.ID,
"upload_time": time.Since(start),
}).Debug("Making relationship")
} }
log.Printf("Uploading all posts for user %s took: %v", username, time.Since(startUploadPosts)) log.WithFields(log.Fields{
"user": e621User.Name,
"id": e621User.ID,
"upload_time": time.Since(startUploadPosts),
}).Info("Upload to Database finished")
return nil return nil
} }
@ -240,6 +276,3 @@ func uploadArtistTagRelationship(ctx context.Context, graphConnection database.G
} }
return nil return nil
} }
//11min für Selloo, simultan mit mutt_jake
//1h58m53 für mutt_jake, mit Selloo