feat: restructure to be a SDK
This commit is contained in:
parent
dbd6617ebe
commit
dcb2b2fc68
2
.gitignore
vendored
2
.gitignore
vendored
@ -190,5 +190,5 @@ $RECYCLE.BIN/
|
|||||||
.idea/*
|
.idea/*
|
||||||
/.run/*
|
/.run/*
|
||||||
*.env
|
*.env
|
||||||
api/gRPC/*
|
*pb.go
|
||||||
!**/.gitkeep
|
!**/.gitkeep
|
16
README.md
16
README.md
@ -1,16 +0,0 @@
|
|||||||
|
|
||||||
After cloning: Look up the following README.md
|
|
||||||
- [Generate gRPC Code](scripts/README.md)
|
|
||||||
- [Build the Plug with Docker](build/package/README.md)
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
# [PLUG_NAME]
|
|
||||||
|
|
||||||
Scrapes users Favorite Posts and Upload them to a graph based database.
|
|
||||||
|
|
||||||
## Environment Variables
|
|
||||||
|
|
||||||
````dotenv
|
|
||||||
|
|
||||||
````
|
|
@ -1,24 +0,0 @@
|
|||||||
FROM golang:alpine AS builder
|
|
||||||
|
|
||||||
ARG PLUG_NAME
|
|
||||||
|
|
||||||
WORKDIR /go/src/git.dragse.it/anthrove/plug-$PLUG_NAME
|
|
||||||
|
|
||||||
RUN apk add -U --no-cache ca-certificates && update-ca-certificates
|
|
||||||
|
|
||||||
COPY go.mod ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . ./
|
|
||||||
|
|
||||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags "-w -s" -o /app ./cmd/
|
|
||||||
|
|
||||||
FROM scratch
|
|
||||||
|
|
||||||
WORKDIR /
|
|
||||||
|
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
|
||||||
COPY --from=builder /app ./
|
|
||||||
|
|
||||||
EXPOSE 8080
|
|
||||||
CMD ["/app"]
|
|
@ -1,34 +0,0 @@
|
|||||||
# Building a Plug
|
|
||||||
|
|
||||||
When building a Plug use the following command and set the value of the ``PLUG_NAME`` with the plugs name. It is also im ortend that you are in the root folder of the project.
|
|
||||||
````bash
|
|
||||||
docker build --build-arg="PLUG_NAME=" -t test -f .\build\package\Dockerfile .
|
|
||||||
````
|
|
||||||
|
|
||||||
Dockerfile:
|
|
||||||
````dockerfile
|
|
||||||
FROM golang:alpine AS builder
|
|
||||||
|
|
||||||
ARG PLUG_NAME
|
|
||||||
|
|
||||||
WORKDIR /go/src/git.dragse.it/anthrove/plug-$PLUG_NAME
|
|
||||||
|
|
||||||
RUN apk add -U --no-cache ca-certificates && update-ca-certificates
|
|
||||||
|
|
||||||
COPY go.mod ./
|
|
||||||
RUN go mod download
|
|
||||||
|
|
||||||
COPY . ./
|
|
||||||
|
|
||||||
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags "-w -s" -o /app ./cmd/
|
|
||||||
|
|
||||||
FROM scratch
|
|
||||||
|
|
||||||
WORKDIR /
|
|
||||||
|
|
||||||
COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
|
||||||
COPY --from=builder /app ./
|
|
||||||
|
|
||||||
EXPOSE 8080
|
|
||||||
CMD ["/app"]
|
|
||||||
````
|
|
37
cmd/main.go
37
cmd/main.go
@ -1,37 +0,0 @@
|
|||||||
package main
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
|
|
||||||
"git.dragse.it/anthrove/plug-template/config"
|
|
||||||
"git.dragse.it/anthrove/plug-template/internal/database/graph"
|
|
||||||
"git.dragse.it/anthrove/plug-template/internal/utils"
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// cfg holds the orchestrator configuration
|
|
||||||
var cfg config.PlugConfig
|
|
||||||
|
|
||||||
// init is used to load the orchestrator configuration
|
|
||||||
func init() {
|
|
||||||
// Load the orchestrator configuration
|
|
||||||
localCfg, err := config.LoadConfig(cfg)
|
|
||||||
if err != nil {
|
|
||||||
log.Panic(err)
|
|
||||||
}
|
|
||||||
cfg = localCfg
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
|
||||||
// ctx is the root context for the main function
|
|
||||||
var ctx = context.Background()
|
|
||||||
|
|
||||||
// Set up the logger based on the configuration
|
|
||||||
utils.SetupLogger(cfg.LogLevel, cfg.LogFormat)
|
|
||||||
|
|
||||||
// Connect to the graph database
|
|
||||||
_, err := graph.ConnectToGraphDatabase(ctx)
|
|
||||||
if err != nil {
|
|
||||||
log.Panicf("main: %v", err)
|
|
||||||
}
|
|
||||||
}
|
|
@ -1,36 +0,0 @@
|
|||||||
package config
|
|
||||||
|
|
||||||
import (
|
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/caarlos0/env/v10"
|
|
||||||
"github.com/go-playground/validator/v10"
|
|
||||||
)
|
|
||||||
|
|
||||||
type PlugConfig struct {
|
|
||||||
LogLevel string `validate:"required,eq=FATAL|eq=ERROR|eq=WARN|eq=INFO|eq=DEBUG|eq=TRACE" env:"LOG_LEVEL" envDefault:"INFO" json:"log_level"`
|
|
||||||
LogFormat string `validate:"required,eq=PLAIN|eq=JSON" env:"LOG_FORMAT" envDefault:"PLAIN" json:"log_format"`
|
|
||||||
}
|
|
||||||
|
|
||||||
type OtterSpaceConfig struct {
|
|
||||||
Type string `env:"DB_TYPE,required"`
|
|
||||||
Endpoint string `env:"DB_URL,required"`
|
|
||||||
Username string `env:"DB_USERNAME,required"`
|
|
||||||
Password string `env:"DB_PASSWORD,required,unset"`
|
|
||||||
Debug bool `env:"DB_DEBUG" envDefault:"FALSE"`
|
|
||||||
}
|
|
||||||
|
|
||||||
// LoadConfig loads the configuration from environment variables and validates it.
|
|
||||||
func LoadConfig[T any](cfg T) (T, error) {
|
|
||||||
|
|
||||||
if err := env.Parse(&cfg); err != nil {
|
|
||||||
return cfg, fmt.Errorf("config: error parsing configuration: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
validate := validator.New()
|
|
||||||
if err := validate.Struct(cfg); err != nil {
|
|
||||||
return cfg, fmt.Errorf("config: validation error: %w", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
return cfg, nil
|
|
||||||
}
|
|
@ -1 +0,0 @@
|
|||||||
|
|
@ -1,12 +0,0 @@
|
|||||||
version: '3.7'
|
|
||||||
|
|
||||||
services:
|
|
||||||
plug:
|
|
||||||
image: anthrove/plug-[PLUG_NAME]:latest
|
|
||||||
build:
|
|
||||||
context: ../.
|
|
||||||
dockerfile: build/package/Dockerfile
|
|
||||||
env_file: plug.env
|
|
||||||
restart: on-failure
|
|
||||||
ports:
|
|
||||||
- "8080:8080"
|
|
13
go.mod
13
go.mod
@ -1,24 +1,19 @@
|
|||||||
module git.dragse.it/anthrove/plug-template
|
module git.dragse.it/anthrove/plug-sdk
|
||||||
|
|
||||||
go 1.22.0
|
go 1.22.0
|
||||||
|
|
||||||
require (
|
require (
|
||||||
git.dragse.it/anthrove/otter-space-sdk v1.0.0
|
git.dragse.it/anthrove/otter-space-sdk v1.0.0
|
||||||
github.com/caarlos0/env/v10 v10.0.0
|
github.com/matoous/go-nanoid/v2 v2.0.0
|
||||||
github.com/go-playground/validator/v10 v10.18.0
|
|
||||||
github.com/sirupsen/logrus v1.9.3
|
|
||||||
google.golang.org/grpc v1.61.1
|
google.golang.org/grpc v1.61.1
|
||||||
google.golang.org/protobuf v1.32.0
|
google.golang.org/protobuf v1.32.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/gabriel-vasile/mimetype v1.4.3 // indirect
|
|
||||||
github.com/go-playground/locales v0.14.1 // indirect
|
|
||||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
|
||||||
github.com/golang/protobuf v1.5.3 // indirect
|
github.com/golang/protobuf v1.5.3 // indirect
|
||||||
github.com/leodido/go-urn v1.4.0 // indirect
|
|
||||||
github.com/neo4j/neo4j-go-driver/v5 v5.17.0 // indirect
|
github.com/neo4j/neo4j-go-driver/v5 v5.17.0 // indirect
|
||||||
golang.org/x/crypto v0.19.0 // indirect
|
github.com/sirupsen/logrus v1.9.3 // indirect
|
||||||
|
github.com/stretchr/testify v1.8.4 // indirect
|
||||||
golang.org/x/net v0.21.0 // indirect
|
golang.org/x/net v0.21.0 // indirect
|
||||||
golang.org/x/sys v0.17.0 // indirect
|
golang.org/x/sys v0.17.0 // indirect
|
||||||
golang.org/x/text v0.14.0 // indirect
|
golang.org/x/text v0.14.0 // indirect
|
||||||
|
20
go.sum
20
go.sum
@ -1,28 +1,17 @@
|
|||||||
git.dragse.it/anthrove/otter-space-sdk v1.0.0 h1:sJ5d8yMFAosP9+315PxZc5yzYUdzAJEEyxoKDmc8YoE=
|
git.dragse.it/anthrove/otter-space-sdk v1.0.0 h1:sJ5d8yMFAosP9+315PxZc5yzYUdzAJEEyxoKDmc8YoE=
|
||||||
git.dragse.it/anthrove/otter-space-sdk v1.0.0/go.mod h1:QI2W7gm1GflcAeipEoyLZ3hj86DlIgwT9Lisc4c99Vs=
|
git.dragse.it/anthrove/otter-space-sdk v1.0.0/go.mod h1:QI2W7gm1GflcAeipEoyLZ3hj86DlIgwT9Lisc4c99Vs=
|
||||||
github.com/caarlos0/env/v10 v10.0.0 h1:yIHUBZGsyqCnpTkbjk8asUlx6RFhhEs+h7TOBdgdzXA=
|
|
||||||
github.com/caarlos0/env/v10 v10.0.0/go.mod h1:ZfulV76NvVPw3tm591U4SwL3Xx9ldzBP9aGxzeN7G18=
|
|
||||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||||
github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
|
|
||||||
github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
|
|
||||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
|
||||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
|
||||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
|
||||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
|
||||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
|
||||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
|
||||||
github.com/go-playground/validator/v10 v10.18.0 h1:BvolUXjp4zuvkZ5YN5t7ebzbhlUtPsPm2S9NAZ5nl9U=
|
|
||||||
github.com/go-playground/validator/v10 v10.18.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
|
|
||||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||||
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
|
||||||
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||||
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
||||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
|
github.com/matoous/go-nanoid v1.5.0/go.mod h1:zyD2a71IubI24efhpvkJz+ZwfwagzgSO6UNiFsZKN7U=
|
||||||
github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
|
github.com/matoous/go-nanoid/v2 v2.0.0 h1:d19kur2QuLeHmJBkvYkFdhFBzLoo1XVm2GgTpL+9Tj0=
|
||||||
|
github.com/matoous/go-nanoid/v2 v2.0.0/go.mod h1:FtS4aGPVfEkxKxhdWPAspZpZSh1cOjtM7Ej/So3hR0g=
|
||||||
github.com/neo4j/neo4j-go-driver/v5 v5.17.0 h1:Bdqg1Y8Hd3uLYToXtBjysDYXTdMiP7zeUNUEwfbJkSo=
|
github.com/neo4j/neo4j-go-driver/v5 v5.17.0 h1:Bdqg1Y8Hd3uLYToXtBjysDYXTdMiP7zeUNUEwfbJkSo=
|
||||||
github.com/neo4j/neo4j-go-driver/v5 v5.17.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
|
github.com/neo4j/neo4j-go-driver/v5 v5.17.0/go.mod h1:Vff8OwT7QpLm7L2yYr85XNWe9Rbqlbeb9asNXJTHO4k=
|
||||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||||
@ -30,11 +19,10 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
|
|||||||
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
|
||||||
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
|
||||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||||
|
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||||
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
|
||||||
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
|
||||||
golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo=
|
|
||||||
golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
|
|
||||||
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||||
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||||
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||||
|
@ -1,47 +0,0 @@
|
|||||||
package graph
|
|
||||||
|
|
||||||
import (
|
|
||||||
"context"
|
|
||||||
"fmt"
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
|
|
||||||
"git.dragse.it/anthrove/plug-template/config"
|
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
)
|
|
||||||
|
|
||||||
var cfg config.OtterSpaceConfig
|
|
||||||
|
|
||||||
func init() {
|
|
||||||
localCfg, err := config.LoadConfig[config.OtterSpaceConfig](cfg)
|
|
||||||
if err != nil {
|
|
||||||
log.Panic(err)
|
|
||||||
}
|
|
||||||
cfg = localCfg
|
|
||||||
}
|
|
||||||
|
|
||||||
// ConnectToGraphDatabase establishes a connection to a graph database specified in the config.
|
|
||||||
// Currently, it only supports Neo4j. If the database type is not supported, it returns an error.
|
|
||||||
func ConnectToGraphDatabase(ctx context.Context) (graph.OtterSpace, error) {
|
|
||||||
var connection graph.OtterSpace
|
|
||||||
var err error
|
|
||||||
|
|
||||||
switch strings.ToLower(cfg.Type) {
|
|
||||||
case "neo4j":
|
|
||||||
connection = graph.NewGraphConnection(cfg.Debug)
|
|
||||||
err = connection.Connect(ctx, cfg.Endpoint, cfg.Username, cfg.Password)
|
|
||||||
if err != nil {
|
|
||||||
return nil, fmt.Errorf("graph: failed to connect to Neo4j: %w", err)
|
|
||||||
}
|
|
||||||
default:
|
|
||||||
return nil, fmt.Errorf("graph: unsupported database type: %s", cfg.Type)
|
|
||||||
}
|
|
||||||
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"graph_type": cfg.Type,
|
|
||||||
"graph_endpoint": cfg.Endpoint,
|
|
||||||
},
|
|
||||||
).Info("graph: successfully connected")
|
|
||||||
return connection, nil
|
|
||||||
}
|
|
@ -1,60 +0,0 @@
|
|||||||
package utils
|
|
||||||
|
|
||||||
import (
|
|
||||||
"strings"
|
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
|
||||||
)
|
|
||||||
|
|
||||||
// SetupLogger configures the logging level and format based on the provided input.
|
|
||||||
// It supports various log levels and two formats: plain text and JSON.
|
|
||||||
func SetupLogger(logLevel string, logFormat string) {
|
|
||||||
setLogLevel(logLevel)
|
|
||||||
setLogFormat(logFormat)
|
|
||||||
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"log_level": log.GetLevel(),
|
|
||||||
"formatter": logFormat,
|
|
||||||
}).Info("Logger configuration set")
|
|
||||||
}
|
|
||||||
|
|
||||||
// setLogLevel configures the logging level based on the provided input.
|
|
||||||
func setLogLevel(logLevel string) {
|
|
||||||
switch strings.ToUpper(logLevel) {
|
|
||||||
case "FATAL":
|
|
||||||
log.SetLevel(log.FatalLevel)
|
|
||||||
case "ERROR":
|
|
||||||
log.SetLevel(log.ErrorLevel)
|
|
||||||
case "WARN":
|
|
||||||
log.SetLevel(log.WarnLevel)
|
|
||||||
case "INFO":
|
|
||||||
log.SetLevel(log.InfoLevel)
|
|
||||||
case "DEBUG":
|
|
||||||
log.SetLevel(log.DebugLevel)
|
|
||||||
case "TRACE":
|
|
||||||
log.SetLevel(log.TraceLevel)
|
|
||||||
default:
|
|
||||||
log.Panic("No valid log level was set")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// setLogFormat configures the logging format based on the provided input.
|
|
||||||
func setLogFormat(logFormat string) {
|
|
||||||
switch strings.ToUpper(logFormat) {
|
|
||||||
case "PLAIN":
|
|
||||||
log.SetFormatter(&log.TextFormatter{
|
|
||||||
ForceColors: true,
|
|
||||||
ForceQuote: true,
|
|
||||||
DisableLevelTruncation: true,
|
|
||||||
PadLevelText: true,
|
|
||||||
FullTimestamp: true,
|
|
||||||
TimestampFormat: "2006-01-02 15:04:05",
|
|
||||||
})
|
|
||||||
case "JSON":
|
|
||||||
log.SetFormatter(&log.JSONFormatter{
|
|
||||||
TimestampFormat: "2006-01-02 15:04:05",
|
|
||||||
})
|
|
||||||
default:
|
|
||||||
log.Panic("No valid formatter was set")
|
|
||||||
}
|
|
||||||
}
|
|
20
pkg/otter/connect.go
Normal file
20
pkg/otter/connect.go
Normal file
@ -0,0 +1,20 @@
|
|||||||
|
package otter
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
|
||||||
|
)
|
||||||
|
|
||||||
|
func ConnectToDatabase(ctx context.Context, endpoint string, username string, password string, debug bool) (graph.OtterSpace, error) {
|
||||||
|
var graphConnection graph.OtterSpace
|
||||||
|
var err error
|
||||||
|
|
||||||
|
graphConnection = graph.NewGraphConnection(debug)
|
||||||
|
err = graphConnection.Connect(ctx, endpoint, username, password)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return graphConnection, err
|
||||||
|
}
|
92
pkg/plug/grpc.go
Normal file
92
pkg/plug/grpc.go
Normal file
@ -0,0 +1,92 @@
|
|||||||
|
package plug
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
|
||||||
|
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
|
||||||
|
"git.dragse.it/anthrove/otter-space-sdk/pkg/models"
|
||||||
|
gRPC "git.dragse.it/anthrove/plug-sdk/pkg/grpc"
|
||||||
|
gonanoid "github.com/matoous/go-nanoid/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
type server struct {
|
||||||
|
gRPC.UnimplementedPlugConnectorServer
|
||||||
|
ctx map[string]context.CancelFunc
|
||||||
|
database graph.OtterSpace
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewGrpcServer(database graph.OtterSpace) gRPC.PlugConnectorServer {
|
||||||
|
return &server{
|
||||||
|
ctx: make(map[string]context.CancelFunc),
|
||||||
|
database: database,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *server) TaskStart(ctx context.Context, creation *gRPC.PlugTaskCreation) (*gRPC.PlugTaskStatus, error) {
|
||||||
|
var anthroveUser models.AnthroveUser
|
||||||
|
var plugTaskState gRPC.PlugTaskStatus
|
||||||
|
var err error
|
||||||
|
|
||||||
|
id, err := gonanoid.New()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
plugTaskState.TaskId = id
|
||||||
|
plugTaskState.TaskState = gRPC.PlugTaskState_RUNNING
|
||||||
|
|
||||||
|
anthroveUser.UserID = models.AnthroveUserID(creation.UserId)
|
||||||
|
|
||||||
|
// gRPC closes the context after the call ended. So the whole scrapping stopped without waiting
|
||||||
|
// by using this method we assign a new context to each new request we get.
|
||||||
|
// This can be used for example to close the context with the given id
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
s.ctx[id] = cancel
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
// FIXME: better implement this methode, works for now but needs refactoring
|
||||||
|
err := service.ScrapeUser(ctx, s.database, &s.e621Client, creation.UserSourceName, &anthroveUser, creation.DeepScrape, func() {
|
||||||
|
s.removeTask(id)
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
log.Print(err)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
return &plugTaskState, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *server) TaskStatus(_ context.Context, task *gRPC.PlugTask) (*gRPC.PlugTaskStatus, error) {
|
||||||
|
var plugTaskState gRPC.PlugTaskStatus
|
||||||
|
|
||||||
|
_, found := s.ctx[task.TaskId]
|
||||||
|
plugTaskState.TaskId = task.TaskId
|
||||||
|
|
||||||
|
plugTaskState.TaskState = gRPC.PlugTaskState_RUNNING
|
||||||
|
|
||||||
|
if !found {
|
||||||
|
plugTaskState.TaskState = gRPC.PlugTaskState_UNKNOWN
|
||||||
|
}
|
||||||
|
return &plugTaskState, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *server) TaskCancel(_ context.Context, task *gRPC.PlugTask) (*gRPC.PlugTaskStatus, error) {
|
||||||
|
var plugTaskState gRPC.PlugTaskStatus
|
||||||
|
|
||||||
|
plugTaskState.TaskState = gRPC.PlugTaskState_STOPPED
|
||||||
|
plugTaskState.TaskId = task.TaskId
|
||||||
|
|
||||||
|
s.removeTask(task.TaskId)
|
||||||
|
|
||||||
|
return &plugTaskState, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *server) removeTask(taskID string) {
|
||||||
|
fn, exists := s.ctx[taskID]
|
||||||
|
if !exists {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fn()
|
||||||
|
delete(s.ctx, taskID)
|
||||||
|
}
|
50
pkg/plug/server.go
Normal file
50
pkg/plug/server.go
Normal file
@ -0,0 +1,50 @@
|
|||||||
|
package plug
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"net"
|
||||||
|
|
||||||
|
"git.dragse.it/anthrove/otter-space-sdk/pkg/graph"
|
||||||
|
pb "git.dragse.it/anthrove/plug-sdk/pkg/grpc"
|
||||||
|
"google.golang.org/grpc"
|
||||||
|
)
|
||||||
|
|
||||||
|
type Server struct {
|
||||||
|
ctx context.Context
|
||||||
|
address string
|
||||||
|
port string
|
||||||
|
database graph.OtterSpace
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewServer(ctx context.Context, address string, port string) Server {
|
||||||
|
return Server{
|
||||||
|
ctx: ctx,
|
||||||
|
address: address,
|
||||||
|
port: port,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Server) Listen() error {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
lis, err := net.Listen("tcp", fmt.Sprintf("%s:%s", s.address, s.port))
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
grpcServer := grpc.NewServer()
|
||||||
|
|
||||||
|
pb.RegisterPlugConnectorServer(grpcServer, NewGrpcServer(s.database))
|
||||||
|
|
||||||
|
err = grpcServer.Serve(lis)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s Server) WithGraphConnection(graph graph.OtterSpace) {
|
||||||
|
s.database = graph
|
||||||
|
}
|
@ -1,28 +1,10 @@
|
|||||||
New-Item -Path "api" -Name "gRPC" -ItemType Directory -Force
|
New-Item -Path "pkg" -Name "grpc" -ItemType Directory -Force
|
||||||
|
|
||||||
$plugName = ""
|
|
||||||
|
|
||||||
protoc `
|
protoc `
|
||||||
--proto_path=third_party/grpc-proto `
|
--proto_path=third_party/grpc-proto `
|
||||||
--go_out=api/gRPC `
|
--go_out=pkg/grpc `
|
||||||
--go_opt=paths=source_relative `
|
--go_opt=paths=source_relative `
|
||||||
--go-grpc_out=api/gRPC `
|
--go-grpc_out=pkg/grpc `
|
||||||
--go-grpc_opt=paths=source_relative `
|
--go-grpc_opt=paths=source_relative `
|
||||||
third_party/grpc-proto/*.proto `
|
third_party/grpc-proto/*.proto `
|
||||||
|
|
||||||
|
|
||||||
function Replace-Text {
|
|
||||||
param (
|
|
||||||
[string]$Path,
|
|
||||||
[string]$OldText,
|
|
||||||
[string]$NewText
|
|
||||||
)
|
|
||||||
|
|
||||||
Get-ChildItem -Path $Path -Recurse -File | ForEach-Object {
|
|
||||||
$content = Get-Content $_.FullName
|
|
||||||
$content = $content -replace $OldText, $NewText
|
|
||||||
Set-Content -Path $_.FullName -Value $content
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
Replace-Text -Path "./api/gRPC" -OldText "git.dragse.it/anthrove/plug-[REPLACE_ME]/api/gRPC" -NewText "git.dragse.it/anthrove/plug-$plugName/api/gRPC"
|
|
||||||
|
@ -1,19 +1,12 @@
|
|||||||
#!/bin/bash
|
#!/bin/bash
|
||||||
|
|
||||||
mkdir -p "api/gRPC"
|
mkdir -p "pkg/grpc"
|
||||||
|
|
||||||
plugName = ""
|
|
||||||
|
|
||||||
protoc \
|
protoc \
|
||||||
--proto_path=third_party/grpc-proto \
|
--proto_path=third_party/grpc-proto \
|
||||||
--go_out=api/gRPC \
|
--go_out=pkg/grpc \
|
||||||
--go_opt=paths=source_relative \
|
--go_opt=paths=source_relative \
|
||||||
--go-grpc_out=api/gRPC \
|
--go-grpc_out=pkg/grpc \
|
||||||
--go-grpc_opt=paths=source_relative \
|
--go-grpc_opt=paths=source_relative \
|
||||||
third_party/grpc-proto/*.proto
|
third_party/grpc-proto/*.proto
|
||||||
|
|
||||||
dir_path="./api/gRPC"
|
|
||||||
old_text="git.dragse.it/anthrove/plug-[REPLACE_ME]/api/gRPC"
|
|
||||||
new_text="git.dragse.it/anthrove/plug-$plugName/api/gRPC""
|
|
||||||
|
|
||||||
grep -rlZ "$old_text" "$dir_path" | xargs -0 sed -i "s#$old_text#$new_text#g"
|
|
||||||
|
Reference in New Issue
Block a user