using new logging provider
This commit is contained in:
David Janowski
parent
f95e9b0853
commit
d9436c5843
|
|
@ -4,7 +4,6 @@ import (
|
|||
"context"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"log"
|
||||
)
|
||||
|
||||
func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error {
|
||||
|
|
@ -21,7 +20,5 @@ func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user mo
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Println("User node created successfully!")
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ package e621
|
|||
import (
|
||||
"e621_to_neo4j/e621/models"
|
||||
"fmt"
|
||||
"log"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// GetFavorites retrieves all favorites from the e621 API.
|
||||
|
|
@ -14,6 +14,11 @@ func (c *Client) GetFavorites(user models.E621User) ([]models.Post, error) {
|
|||
|
||||
for {
|
||||
URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page)
|
||||
log.WithFields(log.Fields{
|
||||
"id": user.ID,
|
||||
"fav_page": page,
|
||||
"uri": URIPath,
|
||||
}).Debug("Requesting API for favorites")
|
||||
favorite, err := ExecuteGetAPIRequest[models.PostResponseWrapper](c, URIPath)
|
||||
if err != nil {
|
||||
log.Printf(err.Error())
|
||||
|
|
|
|||
|
|
@ -5,14 +5,12 @@ import (
|
|||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) {
|
||||
var err error
|
||||
c.limiter.Wait(context.Background())
|
||||
log.Println(URIPath)
|
||||
url := fmt.Sprintf("%s/%s", baseURL, URIPath)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -3,11 +3,16 @@ package e621
|
|||
import (
|
||||
"e621_to_neo4j/e621/models"
|
||||
"fmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// GetUserInfo retrieves the users information from e621 API.
|
||||
func (c *Client) GetUserInfo(username string) (models.E621User, error) {
|
||||
URIPath := fmt.Sprintf("users/%s.json", username)
|
||||
log.WithFields(log.Fields{
|
||||
"username": username,
|
||||
"uri": URIPath,
|
||||
}).Debug("Requesting API for user details")
|
||||
user, err := ExecuteGetAPIRequest[models.E621User](c, URIPath)
|
||||
if err != nil {
|
||||
return models.E621User{}, err
|
||||
|
|
|
|||
19
main.go
19
main.go
|
|
@ -7,12 +7,29 @@ import (
|
|||
neo4j "e621_to_neo4j/database/neo4j"
|
||||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/utils"
|
||||
"log"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"net/http"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func init() {
|
||||
// Log as JSON instead of the default ASCII formatter.
|
||||
//log.SetFormatter(&log.JSONFormatter{})
|
||||
|
||||
// Output to stdout instead of the default stderr
|
||||
// Can be any io.Writer, see below for File example
|
||||
log.SetOutput(os.Stdout)
|
||||
|
||||
// Only log the warning severity or above.
|
||||
log.SetLevel(log.InfoLevel)
|
||||
|
||||
// Logging Method Name
|
||||
//log.SetReportCaller(true)
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
var graphConnection database.GraphConnection
|
||||
ctx := context.Background()
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import (
|
|||
"e621_to_neo4j/e621"
|
||||
"e621_to_neo4j/e621/models"
|
||||
"e621_to_neo4j/utils"
|
||||
"log"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"time"
|
||||
)
|
||||
|
||||
|
|
@ -19,32 +19,50 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
|
|||
}
|
||||
|
||||
if e621User.IsBanned {
|
||||
log.Printf("User %s is banned from e621!", e621User.Name)
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
"bann": e621User.IsBanned,
|
||||
}).Info("User is Banned")
|
||||
return nil
|
||||
}
|
||||
|
||||
log.Printf("Processing user: %s with id %d", e621User.Name, e621User.ID)
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
}).Info("Processing user")
|
||||
|
||||
err = graphConnection.UploadUser(ctx, e621User)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
log.Printf("Getting favorites for user %s with id %d", e621User.Name, e621User.ID)
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
}).Info("Getting favorites for user")
|
||||
start := time.Now()
|
||||
userFavorites, err := e621Client.GetFavorites(e621User)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
log.Printf("User %s with id %d has %d favorites. Time took to scrape: %v", e621User.Name, e621User.ID, len(userFavorites), time.Since(start))
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
"post_amount": len(userFavorites),
|
||||
"scrape_time": time.Since(start),
|
||||
}).Info("Getting favorites for user")
|
||||
|
||||
startUploadPosts := time.Now()
|
||||
|
||||
// Uploads all Tags, Posts as Nodes to Neo4j
|
||||
for i, post := range userFavorites {
|
||||
if exists, err := graphConnection.CheckUserToPostLink(ctx, post.ID, e621User.ID); err == nil && exists {
|
||||
log.Printf("No new posts found for user %s with id %d", e621User.Name, e621User.ID)
|
||||
log.Printf("Last Post ID Found: %d", post.ID)
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
"last_post_id": post.ID,
|
||||
}).Info("No new favorites found")
|
||||
break
|
||||
} else if err != nil {
|
||||
return err
|
||||
|
|
@ -55,7 +73,14 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Uploading post for user %s with id %d, %d of %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
"post_number": i,
|
||||
"post_amount": len(userFavorites),
|
||||
"post_id": post.ID,
|
||||
"upload_time": time.Since(start),
|
||||
}).Debug("Uploading post")
|
||||
|
||||
start := time.Now()
|
||||
err = uploadPostToUserRelationship(ctx, graphConnection, post, e621User)
|
||||
|
|
@ -93,9 +118,20 @@ func ScrapeUser(ctx context.Context, graphConnection database.GraphConnection, e
|
|||
log.Fatal(err)
|
||||
return err
|
||||
}
|
||||
log.Printf("Making relationship for user %s with id %d, %d for Post: %d with ID: %d took: %v", e621User.Name, e621User.ID, i, len(userFavorites), post.ID, time.Since(start))
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
"post_number": i,
|
||||
"post_amount": len(userFavorites),
|
||||
"post_id": post.ID,
|
||||
"upload_time": time.Since(start),
|
||||
}).Debug("Making relationship")
|
||||
}
|
||||
log.Printf("Uploading all posts for user %s took: %v", username, time.Since(startUploadPosts))
|
||||
log.WithFields(log.Fields{
|
||||
"user": e621User.Name,
|
||||
"id": e621User.ID,
|
||||
"upload_time": time.Since(startUploadPosts),
|
||||
}).Info("Upload to Database finished")
|
||||
|
||||
return nil
|
||||
}
|
||||
|
|
@ -240,6 +276,3 @@ func uploadArtistTagRelationship(ctx context.Context, graphConnection database.G
|
|||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
//11min für Selloo, simultan mit mutt_jake
|
||||
//1h58m53 für mutt_jake, mit Selloo
|
||||
|
|
|
|||
Reference in New Issue
Block a user