inital commit
This commit is contained in:
David Janowski
commit
e0a312bffc
190
.gitignore
vendored
Normal file
190
.gitignore
vendored
Normal file
|
|
@ -0,0 +1,190 @@
|
|||
# File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig
|
||||
# Created by https://www.toptal.com/developers/gitignore/api/windows,visualstudiocode,goland,go
|
||||
# Edit at https://www.toptal.com/developers/gitignore?templates=windows,visualstudiocode,goland,go
|
||||
|
||||
### Go ###
|
||||
# If you prefer the allow list template instead of the deny list, see community template:
|
||||
# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
|
||||
#
|
||||
# Binaries for programs and plugins
|
||||
*.exe
|
||||
*.exe~
|
||||
*.dll
|
||||
*.so
|
||||
*.dylib
|
||||
|
||||
# Test binary, built with `go test -c`
|
||||
*.test
|
||||
|
||||
# Output of the go coverage tool, specifically when used with LiteIDE
|
||||
*.out
|
||||
|
||||
# Dependency directories (remove the comment below to include it)
|
||||
# vendor/
|
||||
|
||||
# Go workspace file
|
||||
go.work
|
||||
|
||||
### GoLand ###
|
||||
# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider
|
||||
# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839
|
||||
|
||||
# User-specific stuff
|
||||
.idea/**/workspace.xml
|
||||
.idea/**/tasks.xml
|
||||
.idea/**/usage.statistics.xml
|
||||
.idea/**/dictionaries
|
||||
.idea/**/shelf
|
||||
|
||||
# AWS User-specific
|
||||
.idea/**/aws.xml
|
||||
|
||||
# Generated files
|
||||
.idea/**/contentModel.xml
|
||||
|
||||
# Sensitive or high-churn files
|
||||
.idea/**/dataSources/
|
||||
.idea/**/dataSources.ids
|
||||
.idea/**/dataSources.local.xml
|
||||
.idea/**/sqlDataSources.xml
|
||||
.idea/**/dynamic.xml
|
||||
.idea/**/uiDesigner.xml
|
||||
.idea/**/dbnavigator.xml
|
||||
|
||||
# Gradle
|
||||
.idea/**/gradle.xml
|
||||
.idea/**/libraries
|
||||
|
||||
# Gradle and Maven with auto-import
|
||||
# When using Gradle or Maven with auto-import, you should exclude module files,
|
||||
# since they will be recreated, and may cause churn. Uncomment if using
|
||||
# auto-import.
|
||||
# .idea/artifacts
|
||||
# .idea/compiler.xml
|
||||
# .idea/jarRepositories.xml
|
||||
# .idea/modules.xml
|
||||
# .idea/*.iml
|
||||
# .idea/modules
|
||||
# *.iml
|
||||
# *.ipr
|
||||
|
||||
# CMake
|
||||
cmake-build-*/
|
||||
|
||||
# Mongo Explorer plugin
|
||||
.idea/**/mongoSettings.xml
|
||||
|
||||
# File-based project format
|
||||
*.iws
|
||||
|
||||
# IntelliJ
|
||||
out/
|
||||
|
||||
# mpeltonen/sbt-idea plugin
|
||||
.idea_modules/
|
||||
|
||||
# JIRA plugin
|
||||
atlassian-ide-plugin.xml
|
||||
|
||||
# Cursive Clojure plugin
|
||||
.idea/replstate.xml
|
||||
|
||||
# SonarLint plugin
|
||||
.idea/sonarlint/
|
||||
|
||||
# Crashlytics plugin (for Android Studio and IntelliJ)
|
||||
com_crashlytics_export_strings.xml
|
||||
crashlytics.properties
|
||||
crashlytics-build.properties
|
||||
fabric.properties
|
||||
|
||||
# Editor-based Rest Client
|
||||
.idea/httpRequests
|
||||
|
||||
# Android studio 3.1+ serialized cache file
|
||||
.idea/caches/build_file_checksums.ser
|
||||
|
||||
### GoLand Patch ###
|
||||
# Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721
|
||||
|
||||
# *.iml
|
||||
# modules.xml
|
||||
# .idea/misc.xml
|
||||
# *.ipr
|
||||
|
||||
# Sonarlint plugin
|
||||
# https://plugins.jetbrains.com/plugin/7973-sonarlint
|
||||
.idea/**/sonarlint/
|
||||
|
||||
# SonarQube Plugin
|
||||
# https://plugins.jetbrains.com/plugin/7238-sonarqube-community-plugin
|
||||
.idea/**/sonarIssues.xml
|
||||
|
||||
# Markdown Navigator plugin
|
||||
# https://plugins.jetbrains.com/plugin/7896-markdown-navigator-enhanced
|
||||
.idea/**/markdown-navigator.xml
|
||||
.idea/**/markdown-navigator-enh.xml
|
||||
.idea/**/markdown-navigator/
|
||||
|
||||
# Cache file creation bug
|
||||
# See https://youtrack.jetbrains.com/issue/JBR-2257
|
||||
.idea/$CACHE_FILE$
|
||||
|
||||
# CodeStream plugin
|
||||
# https://plugins.jetbrains.com/plugin/12206-codestream
|
||||
.idea/codestream.xml
|
||||
|
||||
# Azure Toolkit for IntelliJ plugin
|
||||
# https://plugins.jetbrains.com/plugin/8053-azure-toolkit-for-intellij
|
||||
.idea/**/azureSettings.xml
|
||||
|
||||
### VisualStudioCode ###
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
!.vscode/*.code-snippets
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
### VisualStudioCode Patch ###
|
||||
# Ignore all local history of files
|
||||
.history
|
||||
.ionide
|
||||
|
||||
### Windows ###
|
||||
# Windows thumbnail cache files
|
||||
Thumbs.db
|
||||
Thumbs.db:encryptable
|
||||
ehthumbs.db
|
||||
ehthumbs_vista.db
|
||||
|
||||
# Dump file
|
||||
*.stackdump
|
||||
|
||||
# Folder config file
|
||||
[Dd]esktop.ini
|
||||
|
||||
# Recycle Bin used on file shares
|
||||
$RECYCLE.BIN/
|
||||
|
||||
# Windows Installer files
|
||||
*.cab
|
||||
*.msi
|
||||
*.msix
|
||||
*.msm
|
||||
*.msp
|
||||
|
||||
# Windows shortcuts
|
||||
*.lnk
|
||||
|
||||
# End of https://www.toptal.com/developers/gitignore/api/windows,visualstudiocode,goland,go
|
||||
|
||||
# Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option)
|
||||
|
||||
.idea/*
|
||||
8
docker-compose.yml
Normal file
8
docker-compose.yml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
version: '2'
|
||||
services:
|
||||
neo4j:
|
||||
image: docker.io/bitnami/neo4j:latest
|
||||
ports:
|
||||
- '7474:7474'
|
||||
- '7473:7473'
|
||||
- '7687:7687'
|
||||
25
e621/api/client.go
Normal file
25
e621/api/client.go
Normal file
|
|
@ -0,0 +1,25 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
)
|
||||
|
||||
const (
|
||||
baseURL = "https://e621.net"
|
||||
)
|
||||
|
||||
// Client represents the e621 API client.
|
||||
type Client struct {
|
||||
apiKey string
|
||||
username string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
// NewClient creates a new e621 API client.
|
||||
func NewClient(apiKey string, username string) *Client {
|
||||
return &Client{
|
||||
apiKey: apiKey,
|
||||
username: username,
|
||||
client: &http.Client{},
|
||||
}
|
||||
}
|
||||
61
e621/api/favorite.go
Normal file
61
e621/api/favorite.go
Normal file
|
|
@ -0,0 +1,61 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GetFavorites retrieves all favorites from the e621 API.
|
||||
func (c *Client) GetFavorites(user string) ([]models.Post, error) {
|
||||
time.Sleep(2 * time.Second)
|
||||
var lastPostID int64
|
||||
var allFavorites []models.Post
|
||||
|
||||
for {
|
||||
url := fmt.Sprintf("%s/posts.json?tags=fav:%s+status:any&page=b%d", baseURL, user, lastPostID)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "FavGetter (by Selloo)")
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.SetBasicAuth(c.username, c.apiKey)
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return nil, fmt.Errorf("failed to retrieve posts: %s", resp.Status)
|
||||
}
|
||||
|
||||
fetchedFavorites, err := models.UnmarshalE621Post(body)
|
||||
if err != nil {
|
||||
log.Printf(err.Error())
|
||||
}
|
||||
// Append the fetched posts to the result slice
|
||||
allFavorites = append(allFavorites, fetchedFavorites.Posts...)
|
||||
|
||||
// If no more posts are returned, return the accumulated favorites
|
||||
if len(fetchedFavorites.Posts) == 0 {
|
||||
return allFavorites, nil
|
||||
}
|
||||
|
||||
// Update the last post ID for the next page request
|
||||
lastPostID = fetchedFavorites.Posts[len(fetchedFavorites.Posts)-1].ID
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
102
e621/api/models/post.go
Normal file
102
e621/api/models/post.go
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
package models
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
func UnmarshalE621Post(data []byte) (E621Post, error) {
|
||||
var r E621Post
|
||||
err := json.Unmarshal(data, &r)
|
||||
return r, err
|
||||
}
|
||||
|
||||
func (r *E621Post) Marshal() ([]byte, error) {
|
||||
return json.Marshal(r)
|
||||
}
|
||||
|
||||
type E621Post struct {
|
||||
Posts []Post `json:"posts"`
|
||||
}
|
||||
|
||||
type Post struct {
|
||||
ID int64 `json:"id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
File File `json:"file"`
|
||||
Preview Preview `json:"preview"`
|
||||
Sample Sample `json:"sample"`
|
||||
Score Score `json:"score"`
|
||||
Tags Tags `json:"tags"`
|
||||
LockedTags []interface{} `json:"locked_tags"`
|
||||
ChangeSeq int64 `json:"change_seq"`
|
||||
Flags Flags `json:"flags"`
|
||||
Rating string `json:"rating"`
|
||||
FavCount int64 `json:"fav_count"`
|
||||
Sources []string `json:"sources"`
|
||||
Pools []interface{} `json:"pools"`
|
||||
Relationships Relationships `json:"relationships"`
|
||||
ApproverID *int64 `json:"approver_id"`
|
||||
UploaderID int64 `json:"uploader_id"`
|
||||
Description string `json:"description"`
|
||||
CommentCount int64 `json:"comment_count"`
|
||||
IsFavorited bool `json:"is_favorited"`
|
||||
HasNotes bool `json:"has_notes"`
|
||||
Duration interface{} `json:"duration"`
|
||||
}
|
||||
|
||||
type File struct {
|
||||
Width int64 `json:"width"`
|
||||
Height int64 `json:"height"`
|
||||
EXT string `json:"ext"`
|
||||
Size int64 `json:"size"`
|
||||
Md5 string `json:"md5"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type Flags struct {
|
||||
Pending bool `json:"pending"`
|
||||
Flagged bool `json:"flagged"`
|
||||
NoteLocked bool `json:"note_locked"`
|
||||
StatusLocked bool `json:"status_locked"`
|
||||
RatingLocked bool `json:"rating_locked"`
|
||||
Deleted bool `json:"deleted"`
|
||||
}
|
||||
|
||||
type Preview struct {
|
||||
Width int64 `json:"width"`
|
||||
Height int64 `json:"height"`
|
||||
URL string `json:"url"`
|
||||
}
|
||||
|
||||
type Relationships struct {
|
||||
ParentID interface{} `json:"parent_id"`
|
||||
HasChildren bool `json:"has_children"`
|
||||
HasActiveChildren bool `json:"has_active_children"`
|
||||
Children []interface{} `json:"children"`
|
||||
}
|
||||
|
||||
type Sample struct {
|
||||
Has bool `json:"has"`
|
||||
Height int64 `json:"height"`
|
||||
Width int64 `json:"width"`
|
||||
URL string `json:"url"`
|
||||
Alternates Alternates `json:"alternates"`
|
||||
}
|
||||
|
||||
type Alternates struct {
|
||||
}
|
||||
|
||||
type Score struct {
|
||||
Up int64 `json:"up"`
|
||||
Down int64 `json:"down"`
|
||||
Total int64 `json:"total"`
|
||||
}
|
||||
|
||||
type Tags struct {
|
||||
General []string `json:"general"`
|
||||
Species []string `json:"species"`
|
||||
Character []string `json:"character"`
|
||||
Copyright []string `json:"copyright"`
|
||||
Artist []string `json:"artist"`
|
||||
Invalid []interface{} `json:"invalid"`
|
||||
Lore []interface{} `json:"lore"`
|
||||
Meta []string `json:"meta"`
|
||||
}
|
||||
83
e621/api/models/user.go
Normal file
83
e621/api/models/user.go
Normal file
|
|
@ -0,0 +1,83 @@
|
|||
package models
|
||||
|
||||
import "encoding/json"
|
||||
|
||||
func UnmarshalE621User(data []byte) (E621User, error) {
|
||||
var r E621User
|
||||
err := json.Unmarshal(data, &r)
|
||||
return r, err
|
||||
}
|
||||
|
||||
func (r *E621User) Marshal() ([]byte, error) {
|
||||
return json.Marshal(r)
|
||||
}
|
||||
|
||||
type E621User struct {
|
||||
WikiPageVersionCount int64 `json:"wiki_page_version_count"`
|
||||
ArtistVersionCount int64 `json:"artist_version_count"`
|
||||
PoolVersionCount int64 `json:"pool_version_count"`
|
||||
ForumPostCount int64 `json:"forum_post_count"`
|
||||
CommentCount int64 `json:"comment_count"`
|
||||
FlagCount int64 `json:"flag_count"`
|
||||
FavoriteCount int64 `json:"favorite_count"`
|
||||
PositiveFeedbackCount int64 `json:"positive_feedback_count"`
|
||||
NeutralFeedbackCount int64 `json:"neutral_feedback_count"`
|
||||
NegativeFeedbackCount int64 `json:"negative_feedback_count"`
|
||||
UploadLimit int64 `json:"upload_limit"`
|
||||
ID int64 `json:"id"`
|
||||
CreatedAt string `json:"created_at"`
|
||||
Name string `json:"name"`
|
||||
Level int64 `json:"level"`
|
||||
BaseUploadLimit int64 `json:"base_upload_limit"`
|
||||
PostUploadCount int64 `json:"post_upload_count"`
|
||||
PostUpdateCount int64 `json:"post_update_count"`
|
||||
NoteUpdateCount int64 `json:"note_update_count"`
|
||||
IsBanned bool `json:"is_banned"`
|
||||
CanApprovePosts bool `json:"can_approve_posts"`
|
||||
CanUploadFree bool `json:"can_upload_free"`
|
||||
LevelString string `json:"level_string"`
|
||||
AvatarID int64 `json:"avatar_id"`
|
||||
ShowAvatars bool `json:"show_avatars"`
|
||||
BlacklistAvatars bool `json:"blacklist_avatars"`
|
||||
BlacklistUsers bool `json:"blacklist_users"`
|
||||
DescriptionCollapsedInitially bool `json:"description_collapsed_initially"`
|
||||
HideComments bool `json:"hide_comments"`
|
||||
ShowHiddenComments bool `json:"show_hidden_comments"`
|
||||
ShowPostStatistics bool `json:"show_post_statistics"`
|
||||
HasMail bool `json:"has_mail"`
|
||||
ReceiveEmailNotifications bool `json:"receive_email_notifications"`
|
||||
EnableKeyboardNavigation bool `json:"enable_keyboard_navigation"`
|
||||
EnablePrivacyMode bool `json:"enable_privacy_mode"`
|
||||
StyleUsernames bool `json:"style_usernames"`
|
||||
EnableAutoComplete bool `json:"enable_auto_complete"`
|
||||
HasSavedSearches bool `json:"has_saved_searches"`
|
||||
DisableCroppedThumbnails bool `json:"disable_cropped_thumbnails"`
|
||||
DisableMobileGestures bool `json:"disable_mobile_gestures"`
|
||||
EnableSafeMode bool `json:"enable_safe_mode"`
|
||||
DisableResponsiveMode bool `json:"disable_responsive_mode"`
|
||||
DisablePostTooltips bool `json:"disable_post_tooltips"`
|
||||
NoFlagging bool `json:"no_flagging"`
|
||||
NoFeedback bool `json:"no_feedback"`
|
||||
DisableUserDmails bool `json:"disable_user_dmails"`
|
||||
EnableCompactUploader bool `json:"enable_compact_uploader"`
|
||||
ReplacementsBeta bool `json:"replacements_beta"`
|
||||
IsBdStaff bool `json:"is_bd_staff"`
|
||||
UpdatedAt string `json:"updated_at"`
|
||||
Email string `json:"email"`
|
||||
LastLoggedInAt string `json:"last_logged_in_at"`
|
||||
LastForumReadAt string `json:"last_forum_read_at"`
|
||||
RecentTags string `json:"recent_tags"`
|
||||
CommentThreshold int64 `json:"comment_threshold"`
|
||||
DefaultImageSize string `json:"default_image_size"`
|
||||
FavoriteTags string `json:"favorite_tags"`
|
||||
BlacklistedTags string `json:"blacklisted_tags"`
|
||||
TimeZone string `json:"time_zone"`
|
||||
PerPage int64 `json:"per_page"`
|
||||
CustomStyle string `json:"custom_style"`
|
||||
APIRegenMultiplier int64 `json:"api_regen_multiplier"`
|
||||
APIBurstLimit int64 `json:"api_burst_limit"`
|
||||
RemainingAPILimit int64 `json:"remaining_api_limit"`
|
||||
StatementTimeout int64 `json:"statement_timeout"`
|
||||
FavoriteLimit int64 `json:"favorite_limit"`
|
||||
TagQueryLimit int64 `json:"tag_query_limit"`
|
||||
}
|
||||
46
e621/api/users.go
Normal file
46
e621/api/users.go
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
package api
|
||||
|
||||
import (
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
)
|
||||
|
||||
// GetUserInfo retrieves the users information from e621 API.
|
||||
func (c *Client) GetUserInfo(user string) (models.E621User, error) {
|
||||
var e621User models.E621User
|
||||
time.Sleep(2 * time.Second)
|
||||
url := fmt.Sprintf("%s/users/%s.json", baseURL, user)
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
return e621User, err
|
||||
}
|
||||
|
||||
req.Header.Set("User-Agent", "FavGetter (by Selloo)")
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.SetBasicAuth(c.username, c.apiKey)
|
||||
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return e621User, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return e621User, err
|
||||
}
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
return e621User, fmt.Errorf("failed to retrieve posts: %s", resp.Status)
|
||||
}
|
||||
|
||||
e621User, err = models.UnmarshalE621User(body)
|
||||
if err != nil {
|
||||
return e621User, err
|
||||
}
|
||||
|
||||
return e621User, nil
|
||||
}
|
||||
14
go.mod
Normal file
14
go.mod
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
module e621_to_neo4j
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/caarlos0/env v3.5.0+incompatible
|
||||
github.com/neo4j/neo4j-go-driver/v5 v5.8.1
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/davecgh/go-spew v1.1.1 // indirect
|
||||
github.com/stretchr/testify v1.5.1 // indirect
|
||||
gopkg.in/yaml.v2 v2.4.0 // indirect
|
||||
)
|
||||
16
go.sum
Normal file
16
go.sum
Normal file
|
|
@ -0,0 +1,16 @@
|
|||
github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs=
|
||||
github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/neo4j/neo4j-go-driver/v5 v5.8.1 h1:IysKg6KJIUgyItmnHRRrt2N8srbd6znMslRW3qQErTQ=
|
||||
github.com/neo4j/neo4j-go-driver/v5 v5.8.1/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
|
||||
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
|
||||
80
main.go
Normal file
80
main.go
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/api"
|
||||
"e621_to_neo4j/neo4j"
|
||||
"e621_to_neo4j/utils"
|
||||
"log"
|
||||
"time"
|
||||
)
|
||||
|
||||
func main() {
|
||||
config, err := utils.LoadConfig()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
|
||||
driver, err := neo4j.NewConnection(config.Neo4jURL, config.Neo4jUsername, config.Neo4jPassword)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
defer driver.Close(ctx)
|
||||
|
||||
e621Client := api.NewClient(config.E621APIKey, config.E621Username)
|
||||
user, err := e621Client.GetUserInfo("selloo")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
err = neo4j.CreateUserNode(ctx, driver, user)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
favs, err := e621Client.GetFavorites(user.Name)
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
start := time.Now()
|
||||
for i, fav := range favs {
|
||||
|
||||
log.Printf("The e621 post with the id %d has %d general Tags, %d character Tags, %d copyright Tags, %d artist Tags.", fav.ID, len(fav.Tags.General), len(fav.Tags.Character), len(fav.Tags.Copyright), len(fav.Tags.Artist))
|
||||
log.Printf("Uploaded Posts: %d", i)
|
||||
|
||||
for _, general := range fav.Tags.General {
|
||||
log.Printf("TagType: General - Tag: %s", general)
|
||||
err := neo4j.CreateTagNode(ctx, driver, general, "general")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, character := range fav.Tags.Character {
|
||||
log.Printf("TagType: Character - Tag: %s", character)
|
||||
err := neo4j.CreateTagNode(ctx, driver, character, "character")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, copyright := range fav.Tags.Copyright {
|
||||
log.Printf("TagType: Copyright - Tag: %s", copyright)
|
||||
err := neo4j.CreateTagNode(ctx, driver, copyright, "copyright")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
for _, artist := range fav.Tags.Artist {
|
||||
log.Printf("TagType: Artist - Tag: %s", artist)
|
||||
err := neo4j.CreateTagNode(ctx, driver, artist, "artist")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
elapsed := time.Since(start)
|
||||
log.Printf("This took %s", elapsed)
|
||||
}
|
||||
15
neo4j/connection.go
Normal file
15
neo4j/connection.go
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
package neo4j
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
)
|
||||
|
||||
func NewConnection(uri string, username string, password string) (neo4j.DriverWithContext, error) {
|
||||
driver, err := neo4j.NewDriverWithContext(uri, neo4j.BasicAuth(username, password, ""))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to create Neo4j driver: %v", err)
|
||||
}
|
||||
return driver, nil
|
||||
|
||||
}
|
||||
24
neo4j/tag.go
Normal file
24
neo4j/tag.go
Normal file
|
|
@ -0,0 +1,24 @@
|
|||
package neo4j
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
)
|
||||
|
||||
func CreateTagNode(ctx context.Context, driver neo4j.DriverWithContext, name string, tagType string) error {
|
||||
query := `
|
||||
MERGE (u:e621Tag {e621Tag: $name, e621TagType: $tagType})
|
||||
RETURN u
|
||||
`
|
||||
params := map[string]interface{}{
|
||||
"name": name,
|
||||
"tagType": tagType,
|
||||
}
|
||||
|
||||
_, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
27
neo4j/user.go
Normal file
27
neo4j/user.go
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
package neo4j
|
||||
|
||||
import (
|
||||
"context"
|
||||
"e621_to_neo4j/e621/api/models"
|
||||
"github.com/neo4j/neo4j-go-driver/v5/neo4j"
|
||||
"log"
|
||||
)
|
||||
|
||||
func CreateUserNode(ctx context.Context, driver neo4j.DriverWithContext, user models.E621User) error {
|
||||
query := `
|
||||
MERGE (u:e621User {e621ID: $id, e621Username: $name})
|
||||
RETURN u
|
||||
`
|
||||
params := map[string]interface{}{
|
||||
"id": user.ID,
|
||||
"name": user.Name,
|
||||
}
|
||||
|
||||
_, err := neo4j.ExecuteQuery(ctx, driver, query, params, neo4j.EagerResultTransformer)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
log.Println("User node created successfully!")
|
||||
return nil
|
||||
}
|
||||
48
utils/config.go
Normal file
48
utils/config.go
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
package utils
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/caarlos0/env"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
E621APIKey string `env:"e621_API_KEY"`
|
||||
E621Username string `env:"e621_USERNAME"`
|
||||
Neo4jURL string `env:"NEO4J_URL"`
|
||||
Neo4jUsername string `env:"NEO4J_USERNAME"`
|
||||
Neo4jPassword string `env:"NEO4J_PASSWORD"`
|
||||
}
|
||||
|
||||
// LoadConfig loads the configuration from environment variables
|
||||
func LoadConfig() (*Config, error) {
|
||||
config := &Config{}
|
||||
if err := env.Parse(config); err != nil {
|
||||
return nil, fmt.Errorf("error parsing configuration: %w", err)
|
||||
}
|
||||
|
||||
if err := ValidateConfig(config); err != nil {
|
||||
return nil, fmt.Errorf("configuration validation failed: %w", err)
|
||||
}
|
||||
|
||||
return config, nil
|
||||
}
|
||||
|
||||
// ValidateConfig checks if all required fields in the configuration are set
|
||||
func ValidateConfig(config *Config) error {
|
||||
if config.E621APIKey == "" {
|
||||
return fmt.Errorf("e621_API_KEY is not set")
|
||||
}
|
||||
if config.E621Username == "" {
|
||||
return fmt.Errorf("e621_USERNAME is not set")
|
||||
}
|
||||
if config.Neo4jURL == "" {
|
||||
return fmt.Errorf("NEO4J_URL is not set")
|
||||
}
|
||||
if config.Neo4jUsername == "" {
|
||||
return fmt.Errorf("NEO4J_USERNAME is not set")
|
||||
}
|
||||
if config.Neo4jPassword == "" {
|
||||
return fmt.Errorf("NEO4J_PASSWORD is not set")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Reference in New Issue
Block a user