From f3837f5e0dd165d46ce233b1b496420e228b515d Mon Sep 17 00:00:00 2001
From: SoXX <soxx@fenpa.ws>
Date: Thu, 9 Nov 2023 16:11:50 +0100
Subject: [PATCH] feat: added low level API for Database exports

Signed-off-by: SoXX <soxx@fenpa.ws>
---
 pkg/e621/endpoints/dbexport.go | 117 +++++++++++++++++++++++++++++++++
 1 file changed, 117 insertions(+)
 create mode 100644 pkg/e621/endpoints/dbexport.go

diff --git a/pkg/e621/endpoints/dbexport.go b/pkg/e621/endpoints/dbexport.go
new file mode 100644
index 0000000..90f94d9
--- /dev/null
+++ b/pkg/e621/endpoints/dbexport.go
@@ -0,0 +1,117 @@
+package endpoints
+
+import (
+	"fmt"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/model"
+	"git.dragse.it/anthrove/e621-sdk-go/pkg/e621/utils"
+	"golang.org/x/net/html"
+	"net/http"
+	"strings"
+)
+
+// GetDBExportList retrieves a list of files available in the e621 database export.
+//
+// This function performs an HTTP GET request to the e621 database export endpoint and parses
+// the HTML content to extract the links to export files with the ".csv.gz" extension.
+//
+// Parameters:
+//   - requestContext: The context for the API request, including the host, user agent, username, and API key.
+//
+// Returns:
+//   - []string: A slice of file names with the ".csv.gz" extension.
+//   - error: An error, if any, encountered during the API request, response handling, or HTML parsing.
+func GetDBExportList(requestContext model.RequestContext) ([]string, error) {
+	// Create a new HTTP GET request.
+	r, err := http.NewRequest("GET", fmt.Sprintf("https://e621.net/db_export/"), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	r.Header.Set("User-Agent", requestContext.UserAgent)
+	r.Header.Add("Accept", "application/json")
+	r.SetBasicAuth(requestContext.Username, requestContext.APIKey)
+
+	// Send the request using the provided http.Client.
+	resp, err := requestContext.Client.Do(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if the HTTP response status code indicates success (2xx range).
+	if resp.StatusCode < 200 || resp.StatusCode > 300 {
+		// If the status code is outside the 2xx range, return an error based on the status code.
+		return nil, utils.StatusCodesToError(resp.StatusCode)
+	}
+
+	// Parse the HTML content
+	tokenizer := html.NewTokenizer(resp.Body)
+	var files []string
+
+	// Iterate through the HTML tokens
+	for {
+		tokenType := tokenizer.Next()
+
+		switch tokenType {
+		case html.ErrorToken:
+			// End of the HTML document
+			return files, nil
+		case html.StartTagToken, html.SelfClosingTagToken:
+			token := tokenizer.Token()
+			if token.Data == "a" {
+				// Check if the anchor tag has an href attribute
+				for _, attr := range token.Attr {
+					if attr.Key == "href" {
+						// Filter out the parent directory link and only add links with ".csv.gz" extension
+						if !strings.HasPrefix(attr.Val, "../") && strings.HasSuffix(attr.Val, ".csv.gz") {
+							files = append(files, attr.Val)
+						}
+						break
+					}
+				}
+			}
+		}
+	}
+}
+
+// GetDBExportFile retrieves a specific file from the e621 database export.
+//
+// This function performs an HTTP GET request to the e621 database export endpoint to fetch a
+// particular file identified by its name.
+//
+// Parameters:
+//   - requestContext: The context for the API request, including the host, user agent, username, and API key.
+//   - file: The name of the file to be fetched from the database export.
+//
+// Returns:
+//   - *http.Response: The HTTP response containing the requested file (probably a csv.gz).
+//   - error: An error, if any, encountered during the API request or response handling.
+func GetDBExportFile(requestContext model.RequestContext, file string) (*http.Response, error) {
+	if file == "" {
+		return nil, fmt.Errorf("no file specified")
+	}
+
+	// Create a new HTTP GET request.
+	r, err := http.NewRequest("GET", fmt.Sprintf("https://e621.net/db_export/%s", file), nil)
+	if err != nil {
+		return nil, err
+	}
+
+	r.Header.Set("User-Agent", requestContext.UserAgent)
+	r.Header.Add("Accept", "application/json")
+	r.SetBasicAuth(requestContext.Username, requestContext.APIKey)
+
+	// Send the request using the provided http.Client.
+	resp, err := requestContext.Client.Do(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check if the HTTP response status code indicates success (2xx range).
+	if resp.StatusCode < 200 || resp.StatusCode > 300 {
+		// If the status code is outside the 2xx range, return an error based on the status code.
+		return nil, utils.StatusCodesToError(resp.StatusCode)
+	}
+
+	return resp, nil
+
+}