feat: restructure to be a SDK

This commit is contained in:
David Janowski 2024-02-20 19:11:34 +01:00
parent dbd6617ebe
commit dcb2b2fc68
19 changed files with 177 additions and 324 deletions

2
.gitignore vendored
View File

@ -190,5 +190,5 @@ $RECYCLE.BIN/
.idea/*
/.run/*
*.env
api/gRPC/*
*pb.go
!**/.gitkeep

View File

@ -1,16 +0,0 @@
After cloning: Look up the following README.md
- [Generate gRPC Code](scripts/README.md)
- [Build the Plug with Docker](build/package/README.md)
---
# [PLUG_NAME]
Scrapes users Favorite Posts and Upload them to a graph based database.
## Environment Variables
````dotenv
````

View File

View File

@ -1,24 +0,0 @@
FROM golang:alpine AS builder
ARG PLUG_NAME
WORKDIR /go/src/git.dragse.it/anthrove/plug-$PLUG_NAME
RUN apk add -U --no-cache ca-certificates && update-ca-certificates
COPY go.mod ./
RUN go mod download
COPY . ./
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags "-w -s" -o /app ./cmd/
FROM scratch
WORKDIR /
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=builder /app ./
EXPOSE 8080
CMD ["/app"]

View File

@ -1,34 +0,0 @@
# Building a Plug
When building a Plug use the following command and set the value of the ``PLUG_NAME`` with the plugs name. It is also im ortend that you are in the root folder of the project.
````bash
docker build --build-arg="PLUG_NAME=" -t test -f .\build\package\Dockerfile .
````
Dockerfile:
````dockerfile
FROM golang:alpine AS builder
ARG PLUG_NAME
WORKDIR /go/src/git.dragse.it/anthrove/plug-$PLUG_NAME
RUN apk add -U --no-cache ca-certificates && update-ca-certificates
COPY go.mod ./
RUN go mod download
COPY . ./
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags "-w -s" -o /app ./cmd/
FROM scratch
WORKDIR /
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=builder /app ./
EXPOSE 8080
CMD ["/app"]
````

View File

@ -1,37 +0,0 @@
package main
import (
"context"
"git.dragse.it/anthrove/plug-template/config"
"git.dragse.it/anthrove/plug-template/internal/database/graph"
"git.dragse.it/anthrove/plug-template/internal/utils"
log "github.com/sirupsen/logrus"
)
// cfg holds the orchestrator configuration
var cfg config.PlugConfig
// init is used to load the orchestrator configuration
func init() {
// Load the orchestrator configuration
localCfg, err := config.LoadConfig(cfg)
if err != nil {
log.Panic(err)
}
cfg = localCfg
}
func main() {
// ctx is the root context for the main function
var ctx = context.Background()
// Set up the logger based on the configuration
utils.SetupLogger(cfg.LogLevel, cfg.LogFormat)
// Connect to the graph database
_, err := graph.ConnectToGraphDatabase(ctx)
if err != nil {
log.Panicf("main: %v", err)
}
}

View File

@ -1,36 +0,0 @@
package config
import (
"fmt"
"github.com/caarlos0/env/v10"
"github.com/go-playground/validator/v10"
)
type PlugConfig struct {
LogLevel string `validate:"required,eq=FATAL|eq=ERROR|eq=WARN|eq=INFO|eq=DEBUG|eq=TRACE" env:"LOG_LEVEL" envDefault:"INFO" json:"log_level"`
LogFormat string `validate:"required,eq=PLAIN|eq=JSON" env:"LOG_FORMAT" envDefault:"PLAIN" json:"log_format"`
}
type OtterSpaceConfig struct {
Type string `env:"DB_TYPE,required"`
Endpoint string `env:"DB_URL,required"`
Username string `env:"DB_USERNAME,required"`
Password string `env:"DB_PASSWORD,required,unset"`
Debug bool `env:"DB_DEBUG" envDefault:"FALSE"`
}
// LoadConfig loads the configuration from environment variables and validates it.
func LoadConfig[T any](cfg T) (T, error) {
if err := env.Parse(&cfg); err != nil {
return cfg, fmt.Errorf("config: error parsing configuration: %w", err)
}
validate := validator.New()
if err := validate.Struct(cfg); err != nil {
return cfg, fmt.Errorf("config: validation error: %w", err)
}
return cfg, nil
}

View File

@ -1 +0,0 @@

View File

@ -1,12 +0,0 @@
version: '3.7'
services:
plug:
image: anthrove/plug-[PLUG_NAME]:latest
build:
context: ../.
dockerfile: build/package/Dockerfile
env_file: plug.env
restart: on-failure
ports:
- "8080:8080"

13
go.mod
View File

@ -1,24 +1,19 @@
module git.dragse.it/anthrove/plug-template
module git.dragse.it/anthrove/plug-sdk
go 1.22.0
require (
git.dragse.it/anthrove/otter-space-sdk v1.0.0
github.com/caarlos0/env/v10 v10.0.0
github.com/go-playground/validator/v10 v10.18.0
github.com/sirupsen/logrus v1.9.3
github.com/matoous/go-nanoid/v2 v2.0.0
google.golang.org/grpc v1.61.1
google.golang.org/protobuf v1.32.0
)
require (
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
github.com/go-playground/locales v0.14.1 // indirect
github.com/go-playground/universal-translator v0.18.1 // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/neo4j/neo4j-go-driver/v5 v5.17.0 // indirect
golang.org/x/crypto v0.19.0 // indirect
github.com/sirupsen/logrus v1.9.3 // indirect
github.com/stretchr/testify v1.8.4 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.17.0 // indirect
golang.org/x/text v0.14.0 // indirect

20
go.sum
View File

@ -1,28 +1,17 @@
git.dragse.it/anthrove/otter-space-sdk v1.0.0 h1:sJ5d8yMFAosP9+315PxZc5yzYUdzAJEEyxoKDmc8YoE=
git.dragse.it/anthrove/otter-space-sdk v1.0.0/go.mod h1:QI2W7gm1GflcAeipEoyLZ3hj86DlIgwT9Lisc4c99Vs=
github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA=
github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U=
github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
github.com/matoous/go-nanoid v1.5.0/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U=
github.com/matoous/go-nanoid/v2 v2.0.0 h1:d19kur2QuLeHmJBkvYkFdhFBzLoo1XVm2GgTpL+9Tj0=
github.com/matoous/go-nanoid/v2 v2.0.0/go.mod h1:FtS4aGPVfEkxKxhdWPAspZpZSh1cOjtM7Ej/So3hR0g=
github.com/neo4j/neo4j-go-driver/v5 v5.17.0 h1:Bdqg1Y8Hd3uLYToXtBjysDYXTdMiP7zeUNUEwfbJkSo=
github.com/neo4j/neo4j-go-driver/v5 v5.17.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@ -30,11 +19,10 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=

View File

@ -1,47 +0,0 @@
package graph
import (
"context"
"fmt"
"strings"
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
"git.dragse.it/anthrove/plug-template/config"
log "github.com/sirupsen/logrus"
)
var cfg config.OtterSpaceConfig
func init() {
localCfg, err := config.LoadConfig[config.OtterSpaceConfig](cfg)
if err != nil {
log.Panic(err)
}
cfg = localCfg
}
// ConnectToGraphDatabase establishes a connection to a graph database specified in the config.
// Currently, it only supports Neo4j. If the database type is not supported, it returns an error.
func ConnectToGraphDatabase(ctx context.Context) (graph.OtterSpace, error) {
var connection graph.OtterSpace
var err error
switch strings.ToLower(cfg.Type) {
case "neo4j":
connection = graph.NewGraphConnection(cfg.Debug)
err = connection.Connect(ctx, cfg.Endpoint, cfg.Username, cfg.Password)
if err != nil {
return nil, fmt.Errorf("graph: failed to connect to Neo4j: %w", err)
}
default:
return nil, fmt.Errorf("graph: unsupported database type: %s", cfg.Type)
}
log.WithFields(log.Fields{
"graph_type": cfg.Type,
"graph_endpoint": cfg.Endpoint,
},
).Info("graph: successfully connected")
return connection, nil
}

View File

@ -1,60 +0,0 @@
package utils
import (
"strings"
log "github.com/sirupsen/logrus"
)
// SetupLogger configures the logging level and format based on the provided input.
// It supports various log levels and two formats: plain text and JSON.
func SetupLogger(logLevel string, logFormat string) {
setLogLevel(logLevel)
setLogFormat(logFormat)
log.WithFields(log.Fields{
"log_level": log.GetLevel(),
"formatter": logFormat,
}).Info("Logger configuration set")
}
// setLogLevel configures the logging level based on the provided input.
func setLogLevel(logLevel string) {
switch strings.ToUpper(logLevel) {
case "FATAL":
log.SetLevel(log.FatalLevel)
case "ERROR":
log.SetLevel(log.ErrorLevel)
case "WARN":
log.SetLevel(log.WarnLevel)
case "INFO":
log.SetLevel(log.InfoLevel)
case "DEBUG":
log.SetLevel(log.DebugLevel)
case "TRACE":
log.SetLevel(log.TraceLevel)
default:
log.Panic("No valid log level was set")
}
}
// setLogFormat configures the logging format based on the provided input.
func setLogFormat(logFormat string) {
switch strings.ToUpper(logFormat) {
case "PLAIN":
log.SetFormatter(&log.TextFormatter{
ForceColors: true,
ForceQuote: true,
DisableLevelTruncation: true,
PadLevelText: true,
FullTimestamp: true,
TimestampFormat: "2006-01-02 15:04:05",
})
case "JSON":
log.SetFormatter(&log.JSONFormatter{
TimestampFormat: "2006-01-02 15:04:05",
})
default:
log.Panic("No valid formatter was set")
}
}

View File

20
pkg/otter/connect.go Normal file
View File

@ -0,0 +1,20 @@
package otter
import (
"context"
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
)
func ConnectToDatabase(ctx context.Context, endpoint string, username string, password string, debug bool) (graph.OtterSpace, error) {
var graphConnection graph.OtterSpace
var err error
graphConnection = graph.NewGraphConnection(debug)
err = graphConnection.Connect(ctx, endpoint, username, password)
if err != nil {
return nil, err
}
return graphConnection, err
}

92
pkg/plug/grpc.go Normal file
View File

@ -0,0 +1,92 @@
package plug
import (
"context"
"log"
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
"git.dragse.it/anthrove/otter-space-sdk/pkg/models"
gRPC "git.dragse.it/anthrove/plug-sdk/pkg/grpc"
gonanoid "github.com/matoous/go-nanoid/v2"
)
type server struct {
gRPC.UnimplementedPlugConnectorServer
ctx map[string]context.CancelFunc
database graph.OtterSpace
}
func NewGrpcServer(database graph.OtterSpace) gRPC.PlugConnectorServer {
return &server{
ctx: make(map[string]context.CancelFunc),
database: database,
}
}
func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation) (*gRPC.PlugTaskStatus, error) {
var anthroveUser models.AnthroveUser
var plugTaskState gRPC.PlugTaskStatus
var err error
id, err := gonanoid.New()
if err != nil {
return nil, err
}
plugTaskState.TaskId = id
plugTaskState.TaskState = gRPC.PlugTaskState_RUNNING
anthroveUser.UserID = models.AnthroveUserID(creation.UserId)
// gRPC closes the context after the call ended. So the whole scrapping stopped without waiting
// by using this method we assign a new context to each new request we get.
// This can be used for example to close the context with the given id
ctx, cancel := context.WithCancel(context.Background())
s.ctx[id] = cancel
go func() {
// FIXME: better implement this methode, works for now but needs refactoring
err := service.ScrapeUser(ctx, s.database, &s.e621Client, creation.UserSourceName, &anthroveUser, creation.DeepScrape, func() {
s.removeTask(id)
})
if err != nil {
log.Print(err)
}
}()
return &plugTaskState, nil
}
func (s *server) TaskStatus(_ context.Context, task *gRPC.PlugTask) (*gRPC.PlugTaskStatus, error) {
var plugTaskState gRPC.PlugTaskStatus
_, found := s.ctx[task.TaskId]
plugTaskState.TaskId = task.TaskId
plugTaskState.TaskState = gRPC.PlugTaskState_RUNNING
if !found {
plugTaskState.TaskState = gRPC.PlugTaskState_UNKNOWN
}
return &plugTaskState, nil
}
func (s *server) TaskCancel(_ context.Context, task *gRPC.PlugTask) (*gRPC.PlugTaskStatus, error) {
var plugTaskState gRPC.PlugTaskStatus
plugTaskState.TaskState = gRPC.PlugTaskState_STOPPED
plugTaskState.TaskId = task.TaskId
s.removeTask(task.TaskId)
return &plugTaskState, nil
}
func (s *server) removeTask(taskID string) {
fn, exists := s.ctx[taskID]
if !exists {
return
}
fn()
delete(s.ctx, taskID)
}

50
pkg/plug/server.go Normal file
View File

@ -0,0 +1,50 @@
package plug
import (
"context"
"fmt"
"net"
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
pb "git.dragse.it/anthrove/plug-sdk/pkg/grpc"
"google.golang.org/grpc"
)
type Server struct {
ctx context.Context
address string
port string
database graph.OtterSpace
}
func NewServer(ctx context.Context, address string, port string) Server {
return Server{
ctx: ctx,
address: address,
port: port,
}
}
func (s Server) Listen() error {
var err error
lis, err := net.Listen("tcp", fmt.Sprintf("%s:%s", s.address, s.port))
if err != nil {
return err
}
grpcServer := grpc.NewServer()
pb.RegisterPlugConnectorServer(grpcServer, NewGrpcServer(s.database))
err = grpcServer.Serve(lis)
if err != nil {
return err
}
return nil
}
func (s Server) WithGraphConnection(graph graph.OtterSpace) {
s.database = graph
}

View File

@ -1,28 +1,10 @@
New-Item -Path "api" -Name "gRPC" -ItemType Directory -Force
$plugName = ""
New-Item -Path "pkg" -Name "grpc" -ItemType Directory -Force
protoc `
--proto_path=third_party/grpc-proto `
--go_out=api/gRPC `
--go_out=pkg/grpc `
--go_opt=paths=source_relative `
--go-grpc_out=api/gRPC `
--go-grpc_out=pkg/grpc `
--go-grpc_opt=paths=source_relative `
third_party/grpc-proto/*.proto `
function Replace-Text {
param (
[string]$Path,
[string]$OldText,
[string]$NewText
)
Get-ChildItem -Path $Path -Recurse -File | ForEach-Object {
$content = Get-Content $_.FullName
$content = $content -replace $OldText, $NewText
Set-Content -Path $_.FullName -Value $content
}
}
Replace-Text -Path "./api/gRPC" -OldText "git.dragse.it/anthrove/plug-[REPLACE_ME]/api/gRPC" -NewText "git.dragse.it/anthrove/plug-$plugName/api/gRPC"

View File

@ -1,19 +1,12 @@
#!/bin/bash
mkdir -p "api/gRPC"
plugName = ""
mkdir -p "pkg/grpc"
protoc \
--proto_path=third_party/grpc-proto \
--go_out=api/gRPC \
--go_out=pkg/grpc \
--go_opt=paths=source_relative \
--go-grpc_out=api/gRPC \
--go-grpc_out=pkg/grpc \
--go-grpc_opt=paths=source_relative \
third_party/grpc-proto/*.proto
dir_path="./api/gRPC"
old_text="git.dragse.it/anthrove/plug-[REPLACE_ME]/api/gRPC"
new_text="git.dragse.it/anthrove/plug-$plugName/api/gRPC""
grep -rlZ "$old_text" "$dir_path" | xargs -0 sed -i "s#$old_text#$new_text#g"