From 3a964f598f788f33dcf4a58fd4a30a9cbcca5e20 Mon Sep 17 00:00:00 2001 From: David Janowski Date: Sat, 17 Jun 2023 20:51:04 +0200 Subject: [PATCH] generalized request --- e621/favorite.go | 40 ++++++++-------------------------------- e621/request.go | 45 +++++++++++++++++++++++++++++++++++++++++++++ e621/users.go | 40 +++++----------------------------------- utils/queue.go | 2 +- utils/queue_test.go | 14 +++++++------- 5 files changed, 66 insertions(+), 75 deletions(-) create mode 100644 e621/request.go diff --git a/e621/favorite.go b/e621/favorite.go index 2d73a81..6083457 100644 --- a/e621/favorite.go +++ b/e621/favorite.go @@ -3,58 +3,34 @@ package e621 import ( "e621_to_neo4j/e621/models" "fmt" - "io" "log" - "net/http" "time" ) // GetFavorites retrieves all favorites from the e621 API. func (c *Client) GetFavorites(user models.E621User) ([]models.Post, error) { time.Sleep(1 * time.Second) - var lastPostID int64 + var page int64 var allFavorites []models.Post - var url string + var URIPath string for { - url = fmt.Sprintf("%s/favorites.json?user_id=%d&page=%d", baseURL, user.ID, lastPostID) - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return nil, err - } - - req.Header.Set("User-Agent", "e621 to GraphDB (by Selloo)") - req.Header.Add("Accept", "application/json") - req.SetBasicAuth(c.username, c.apiKey) - - resp, err := c.client.Do(req) - if err != nil { - return nil, err - } - - body, err := io.ReadAll(resp.Body) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - return nil, fmt.Errorf("failed to retrieve posts: %s", resp.Status) - } - - fetchedFavorites, err := models.UnmarshalE621Post(body) + URIPath = fmt.Sprintf("favorites.json?user_id=%d&page=%d", user.ID, page) + favorite, err := ExecuteGetAPIRequest[models.APIe621Post](c, URIPath) if err != nil { log.Printf(err.Error()) } + // Append the fetched posts to the result slice - allFavorites = append(allFavorites, fetchedFavorites.Posts...) + allFavorites = append(allFavorites, favorite.Posts...) // If no more posts are returned, return the accumulated favorites - if len(fetchedFavorites.Posts) == 0 { + if len(favorite.Posts) == 0 { return allFavorites, nil } // Update the last post ID for the next page request - lastPostID = lastPostID + 1 + page += 1 } } diff --git a/e621/request.go b/e621/request.go new file mode 100644 index 0000000..2ee1668 --- /dev/null +++ b/e621/request.go @@ -0,0 +1,45 @@ +package e621 + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "time" +) + +func ExecuteGetAPIRequest[dataType any](c *Client, URIPath string) (*dataType, error) { + time.Sleep(1 * time.Second) + var err error + url := fmt.Sprintf("%s/%s", baseURL, URIPath) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + req.Header.Set("User-Agent", "FavGetter (by Selloo)") + req.Header.Add("Accept", "application/json") + req.SetBasicAuth(c.username, c.apiKey) + + resp, err := c.client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("failed to retrieve posts: %s", resp.Status) + } + + var r dataType + err = json.Unmarshal(body, &r) + if err != nil { + return nil, err + } + return &r, nil +} diff --git a/e621/users.go b/e621/users.go index c35650a..8d2f3fe 100644 --- a/e621/users.go +++ b/e621/users.go @@ -3,44 +3,14 @@ package e621 import ( "e621_to_neo4j/e621/models" "fmt" - "io" - "net/http" - "time" ) // GetUserInfo retrieves the users information from e621 API. -func (c *Client) GetUserInfo(user string) (models.E621User, error) { - var e621User models.E621User - time.Sleep(2 * time.Second) - url := fmt.Sprintf("%s/users/%s.json", baseURL, user) - req, err := http.NewRequest("GET", url, nil) +func (c *Client) GetUserInfo(username string) (models.E621User, error) { + URIPath := fmt.Sprintf("users/%s.json", username) + user, err := ExecuteGetAPIRequest[models.E621User](c, URIPath) if err != nil { - return e621User, err + return models.E621User{}, err } - - req.Header.Set("User-Agent", "FavGetter (by Selloo)") - req.Header.Add("Accept", "application/json") - req.SetBasicAuth(c.username, c.apiKey) - - resp, err := c.client.Do(req) - if err != nil { - return e621User, err - } - defer resp.Body.Close() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return e621User, err - } - - if resp.StatusCode != http.StatusOK { - return e621User, fmt.Errorf("failed to retrieve posts: %s", resp.Status) - } - - e621User, err = models.UnmarshalE621User(body) - if err != nil { - return e621User, err - } - - return e621User, nil + return *user, nil } diff --git a/utils/queue.go b/utils/queue.go index 2cb865e..a75e575 100644 --- a/utils/queue.go +++ b/utils/queue.go @@ -5,7 +5,7 @@ import ( ) type Task struct { - URL string `json:"url,omitempty" :"url"` + URIPath string `json:"url,omitempty" :"url"` Methode string `json:"method,omitempty" :"method"` Channel chan any `:"channel"` } diff --git a/utils/queue_test.go b/utils/queue_test.go index aac19b9..aaf1b43 100644 --- a/utils/queue_test.go +++ b/utils/queue_test.go @@ -24,12 +24,12 @@ func TestQueue_Pop(t *testing.T) { { name: "Pop element of a filled list with three elements", fields: fields{elements: []Task{ - {URL: "https://e621.net0....", Methode: "GET", Channel: nil}, - {URL: "https://e621.net1....", Methode: "GET", Channel: nil}, - {URL: "https://e621.net2....", Methode: "GET", Channel: nil}, + {URIPath: "https://e621.net0....", Methode: "GET", Channel: nil}, + {URIPath: "https://e621.net1....", Methode: "GET", Channel: nil}, + {URIPath: "https://e621.net2....", Methode: "GET", Channel: nil}, }}, want: Task{ - URL: "https://e621.net0....", + URIPath: "https://e621.net0....", Methode: "GET", Channel: nil, }, @@ -57,7 +57,7 @@ func TestQueue_Push(t *testing.T) { t.Run("Push tasks to empty queue", func(t *testing.T) { queue := Queue{elements: []Task{}} task := Task{ - URL: "http://e621.net0....", + URIPath: "http://e621.net0....", Methode: "GET", Channel: nil, } @@ -74,12 +74,12 @@ func TestQueue_Push(t *testing.T) { }) t.Run("Push tasks to filled queue", func(t *testing.T) { queue := Queue{elements: []Task{{ - URL: "http://e621.net0....", + URIPath: "http://e621.net0....", Methode: "GET", Channel: nil, }}} task := Task{ - URL: "http://e621.net1....", + URIPath: "http://e621.net1....", Methode: "GET", Channel: nil, }