From e91077f174339a12348e9d3f4e9372d307cb3423 Mon Sep 17 00:00:00 2001
From: Dedy Martadinata S <dedyms@proton.me>
Date: Mon, 20 Mar 2023 13:16:46 +0700
Subject: [PATCH] add migration test
---
.github/workflows/docker-build-static.yml | 199 +++++++++++++++++++++-
docs/database_migration.md | 13 +-
2 files changed, 205 insertions(+), 7 deletions(-)
diff --git a/.github/workflows/docker-build-static.yml b/.github/workflows/docker-build-static.yml
index d331127..4ac164a 100644
--- a/.github/workflows/docker-build-static.yml
+++ b/.github/workflows/docker-build-static.yml
@@ -4,6 +4,9 @@ on:
push:
branches:
- 'main'
+ paths:
+ - '!docs/**'
+ - '!example_configs/**'
release:
types:
- 'published'
@@ -164,9 +167,9 @@ jobs:
name: ${{ matrix.target }}-lldap_set_password-bin
path: target/${{ matrix.target }}/release/lldap_set_password
- lldap-database-integration-test:
+ lldap-database-init-test:
needs: [build-ui,build-bin]
- name: LLDAP test
+ name: LLDAP database init test
runs-on: ubuntu-latest
services:
mariadb:
@@ -178,6 +181,7 @@ jobs:
MYSQL_PASSWORD: lldappass
MYSQL_DATABASE: lldap
MYSQL_ROOT_PASSWORD: rootpass
+ options: --name mariadb
postgresql:
image: postgres:latest
@@ -187,6 +191,7 @@ jobs:
POSTGRES_USER: lldapuser
POSTGRES_PASSWORD: lldappass
POSTGRES_DB: lldap
+ options: --name postgresql
steps:
- name: Download artifacts
@@ -194,8 +199,7 @@ jobs:
with:
name: x86_64-unknown-linux-musl-lldap-bin
path: bin/
- - name: Where is the bin?
- run: ls -alR bin
+
- name: Set executables to LLDAP
run: chmod +x bin/lldap
@@ -230,8 +234,193 @@ jobs:
LLDAP_database_url: sqlite://users.db?mode=rwc
LLDAP_ldap_port: 3892
LLDAP_http_port: 17172
+
+ - name: Check DB container logs
+ run: |
+ docker logs -n 20 mariadb
+ docker logs -n 20 postgresql
+ lldap-database-migration-test:
+ needs: [build-ui,build-bin]
+ name: LLDAP database migration test
+ runs-on: ubuntu-latest
+ services:
+ postgresql:
+ image: postgres:latest
+ ports:
+ - 5432:5432
+ env:
+ POSTGRES_USER: lldapuser
+ POSTGRES_PASSWORD: lldappass
+ POSTGRES_DB: lldap
+ options: --name postgresql
+ mariadb:
+ image: mariadb:latest
+ ports:
+ - 3306:3306
+ env:
+ MYSQL_USER: lldapuser
+ MYSQL_PASSWORD: lldappass
+ MYSQL_DATABASE: lldap
+ MYSQL_ROOT_PASSWORD: rootpass
+ options: --name mariadb
+ mysql:
+ image: mysql:latest
+ ports:
+ - 3307:3306
+ env:
+ MYSQL_USER: lldapuser
+ MYSQL_PASSWORD: lldappass
+ MYSQL_DATABASE: lldap
+ MYSQL_ROOT_PASSWORD: rootpass
+ options: --name mysql
+
+ steps:
+ - name: Download LLDAP artifacts
+ uses: actions/download-artifact@v3
+ with:
+ name: x86_64-unknown-linux-musl-lldap-bin
+ path: bin/
+
+ - name: Download LLDAP set password
+ uses: actions/download-artifact@v3
+ with:
+ name: x86_64-unknown-linux-musl-lldap_set_password-bin
+ path: bin/
+
+ - name: Where is the bin?
+ run: ls -alR bin
+
+ - name: Set executables to LLDAP and LLDAP set password
+ run: |
+ chmod +x bin/lldap
+ chmod +x bin/lldap_set_password
+
+ - name: Install sqlite3 and ldap-utils for exporting and searching dummy user
+ run: sudo apt update && sudo apt install -y sqlite3 ldap-utils
+
+ - name: Run lldap with sqlite DB and healthcheck
+ run: |
+ bin/lldap run &
+ sleep 10s
+ bin/lldap healthcheck
+ env:
+ LLDAP_database_url: sqlite://users.db?mode=rwc
+ LLDAP_ldap_port: 3890
+ LLDAP_http_port: 17170
+ LLDAP_LDAP_USER_PASS: ldappass
+ LLDAP_JWT_SECRET: somejwtsecret
+
+ - name: Create dummy user
+ run: |
+ TOKEN=$(curl -X POST -H "Content-Type: application/json" -d '{"username": "admin", "password": "ldappass"}' http://localhost:17170/auth/simple/login | cut -c 11-277)
+ echo "$TOKEN"
+ curl 'http://localhost:17170/api/graphql' -H 'Content-Type: application/json' -H "Authorization: Bearer ${TOKEN//[$'\t\r\n ']}" --data-binary '{"query":"mutation{\n createUser(user:\n {\n id: \"dummyuser\",\n email: \"dummyuser@example.com\"\n }\n )\n {\n id\n email\n }\n}\n\n\n"}' --compressed
+ bin/lldap_set_password --base-url http://localhost:17170 --admin-username admin --admin-password ldappass --token $TOKEN --username dummyuser --password dummypassword
+
+ - name: Test Dummy User, This will be checked again after importing
+ run: |
+ ldapsearch -H ldap://localhost:3890 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
+
+ - name: Stop LLDAP sqlite
+ run: pkill lldap
+
+ - name: Export and Converting to Postgress
+ run: |
+ curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
+ chmod +x ./helper.sh
+ ./helper.sh | sqlite3 ./users.db > ./dump.sql
+ sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" -e '1s/^/BEGIN;\n/' -e '$aCOMMIT;' ./dump.sql
+
+ - name: Create schema on postgres
+ run: |
+ bin/lldap create_schema -d postgres://lldapuser:lldappass@localhost:5432/lldap
+
+ - name: Copy converted db to postgress and import
+ run: |
+ docker ps -a
+ docker cp ./dump.sql postgresql:/tmp/dump.sql
+ docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql"
+ rm ./dump.sql
+
+ - name: Export and Converting to mariadb
+ run: |
+ curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
+ chmod +x ./helper.sh
+ ./helper.sh | sqlite3 ./users.db > ./dump.sql
+ cp ./dump.sql ./dump-no-sed.sql
+ sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
+ sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
+
+ - name: Create schema on mariadb
+ run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3306/lldap
+
+ - name: Copy converted db to mariadb and import
+ run: |
+ docker ps -a
+ docker cp ./dump.sql mariadb:/tmp/dump.sql
+ docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
+ rm ./dump.sql
+
+ - name: Export and Converting to mysql
+ run: |
+ curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
+ chmod +x ./helper.sh
+ ./helper.sh | sqlite3 ./users.db > ./dump.sql
+ sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
+ sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
+
+ - name: Create schema on mysql
+ run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3307/lldap
+
+ - name: Copy converted db to mysql and import
+ run: |
+ docker ps -a
+ docker cp ./dump.sql mysql:/tmp/dump.sql
+ docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
+ rm ./dump.sql
+
+ - name: Run lldap with postgres DB and healthcheck again
+ run: |
+ bin/lldap run &
+ sleep 10s
+ bin/lldap healthcheck
+ env:
+ LLDAP_database_url: postgres://lldapuser:lldappass@localhost:5432/lldap
+ LLDAP_ldap_port: 3891
+ LLDAP_http_port: 17171
+ LLDAP_LDAP_USER_PASS: ldappass
+ LLDAP_JWT_SECRET: somejwtsecret
+
+ - name: Run lldap with mariaDB and healthcheck again
+ run: |
+ bin/lldap run &
+ sleep 10s
+ bin/lldap healthcheck
+ env:
+ LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3306/lldap
+ LLDAP_ldap_port: 3892
+ LLDAP_http_port: 17172
+ LLDAP_JWT_SECRET: somejwtsecret
+
+ - name: Run lldap with mysql and healthcheck again
+ run: |
+ bin/lldap run &
+ sleep 10s
+ bin/lldap healthcheck
+ env:
+ LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3307/lldap
+ LLDAP_ldap_port: 3893
+ LLDAP_http_port: 17173
+ LLDAP_JWT_SECRET: somejwtsecret
+
+ - name: Test Dummy User
+ run: |
+ ldapsearch -H ldap://localhost:3891 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
+ ldapsearch -H ldap://localhost:3892 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
+ ldapsearch -H ldap://localhost:3893 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
+
build-docker-image:
needs: [build-ui, build-bin]
name: Build Docker image
@@ -356,6 +545,8 @@ jobs:
name: Create release artifacts
if: github.event_name == 'release'
runs-on: ubuntu-latest
+ permissions:
+ contents: write
steps:
- name: Download all artifacts
uses: actions/download-artifact@v3
diff --git a/docs/database_migration.md b/docs/database_migration.md
index cf29b81..9538aab 100644
--- a/docs/database_migration.md
+++ b/docs/database_migration.md
@@ -65,7 +65,8 @@ a transaction:
```
sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
-e '1s/^/START TRANSACTION;\n/' \
--e '$aCOMMIT;' /path/to/dump.sql
+-e '$aCOMMIT;' \
+-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql
```
### To MariaDB
@@ -77,7 +78,8 @@ strings. Use the following command to remove those and perform the additional My
sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \
-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
-e '1s/^/START TRANSACTION;\n/' \
--e '$aCOMMIT;' /path/to/dump.sql
+-e '$aCOMMIT;' \
+-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql
```
## Insert data
@@ -102,4 +104,9 @@ or
Modify your `database_url` in `lldap_config.toml` (or `LLDAP_DATABASE_URL` in the env)
to point to your new database (the same value used when generating schema). Restart
-LLDAP and check the logs to ensure there were no errors.
\ No newline at end of file
+LLDAP and check the logs to ensure there were no errors.
+
+### More details/examples can be seen in the CI process [here](https://raw.githubusercontent.com/nitnelave/lldap/main/.github/workflows/docker-build-static.yml)
+* Postgres L308-327
+* Mariadb L328-348
+* MySQL L349-367
\ No newline at end of file