docker: Add DB migration tests in the CI

This commit is contained in:
Dedy Martadinata S 2023-03-28 18:59:23 +07:00 committed by GitHub
parent 84b4c66309
commit c7c6d95334
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 202 additions and 7 deletions

View File

@ -4,12 +4,18 @@ on:
push: push:
branches: branches:
- 'main' - 'main'
paths-ignore:
- 'docs/**'
- 'example_configs/**'
release: release:
types: types:
- 'published' - 'published'
pull_request: pull_request:
branches: branches:
- 'main' - 'main'
paths-ignore:
- 'docs/**'
- 'example_configs/**'
workflow_dispatch: workflow_dispatch:
inputs: inputs:
msg: msg:
@ -164,9 +170,9 @@ jobs:
name: ${{ matrix.target }}-lldap_set_password-bin name: ${{ matrix.target }}-lldap_set_password-bin
path: target/${{ matrix.target }}/release/lldap_set_password path: target/${{ matrix.target }}/release/lldap_set_password
lldap-database-integration-test: lldap-database-init-test:
needs: [build-ui,build-bin] needs: [build-ui,build-bin]
name: LLDAP test name: LLDAP database init test
runs-on: ubuntu-latest runs-on: ubuntu-latest
services: services:
mariadb: mariadb:
@ -178,6 +184,7 @@ jobs:
MYSQL_PASSWORD: lldappass MYSQL_PASSWORD: lldappass
MYSQL_DATABASE: lldap MYSQL_DATABASE: lldap
MYSQL_ROOT_PASSWORD: rootpass MYSQL_ROOT_PASSWORD: rootpass
options: --name mariadb
postgresql: postgresql:
image: postgres:latest image: postgres:latest
@ -187,6 +194,7 @@ jobs:
POSTGRES_USER: lldapuser POSTGRES_USER: lldapuser
POSTGRES_PASSWORD: lldappass POSTGRES_PASSWORD: lldappass
POSTGRES_DB: lldap POSTGRES_DB: lldap
options: --name postgresql
steps: steps:
- name: Download artifacts - name: Download artifacts
@ -194,8 +202,7 @@ jobs:
with: with:
name: x86_64-unknown-linux-musl-lldap-bin name: x86_64-unknown-linux-musl-lldap-bin
path: bin/ path: bin/
- name: Where is the bin?
run: ls -alR bin
- name: Set executables to LLDAP - name: Set executables to LLDAP
run: chmod +x bin/lldap run: chmod +x bin/lldap
@ -231,6 +238,188 @@ jobs:
LLDAP_ldap_port: 3892 LLDAP_ldap_port: 3892
LLDAP_http_port: 17172 LLDAP_http_port: 17172
- name: Check DB container logs
run: |
docker logs -n 20 mariadb
docker logs -n 20 postgresql
lldap-database-migration-test:
needs: [build-ui,build-bin]
name: LLDAP database migration test
runs-on: ubuntu-latest
services:
postgresql:
image: postgres:latest
ports:
- 5432:5432
env:
POSTGRES_USER: lldapuser
POSTGRES_PASSWORD: lldappass
POSTGRES_DB: lldap
options: --name postgresql
mariadb:
image: mariadb:latest
ports:
- 3306:3306
env:
MYSQL_USER: lldapuser
MYSQL_PASSWORD: lldappass
MYSQL_DATABASE: lldap
MYSQL_ROOT_PASSWORD: rootpass
options: --name mariadb
mysql:
image: mysql:latest
ports:
- 3307:3306
env:
MYSQL_USER: lldapuser
MYSQL_PASSWORD: lldappass
MYSQL_DATABASE: lldap
MYSQL_ROOT_PASSWORD: rootpass
options: --name mysql
steps:
- name: Download LLDAP artifacts
uses: actions/download-artifact@v3
with:
name: x86_64-unknown-linux-musl-lldap-bin
path: bin/
- name: Download LLDAP set password
uses: actions/download-artifact@v3
with:
name: x86_64-unknown-linux-musl-lldap_set_password-bin
path: bin/
- name: Set executables to LLDAP and LLDAP set password
run: |
chmod +x bin/lldap
chmod +x bin/lldap_set_password
- name: Install sqlite3 and ldap-utils for exporting and searching dummy user
run: sudo apt update && sudo apt install -y sqlite3 ldap-utils
- name: Run lldap with sqlite DB and healthcheck
run: |
bin/lldap run &
sleep 10s
bin/lldap healthcheck
env:
LLDAP_database_url: sqlite://users.db?mode=rwc
LLDAP_ldap_port: 3890
LLDAP_http_port: 17170
LLDAP_LDAP_USER_PASS: ldappass
LLDAP_JWT_SECRET: somejwtsecret
- name: Create dummy user
run: |
TOKEN=$(curl -X POST -H "Content-Type: application/json" -d '{"username": "admin", "password": "ldappass"}' http://localhost:17170/auth/simple/login | cut -c 11-277)
echo "$TOKEN"
curl 'http://localhost:17170/api/graphql' -H 'Content-Type: application/json' -H "Authorization: Bearer ${TOKEN//[$'\t\r\n ']}" --data-binary '{"query":"mutation{\n createUser(user:\n {\n id: \"dummyuser\",\n email: \"dummyuser@example.com\"\n }\n )\n {\n id\n email\n }\n}\n\n\n"}' --compressed
bin/lldap_set_password --base-url http://localhost:17170 --admin-username admin --admin-password ldappass --token $TOKEN --username dummyuser --password dummypassword
- name: Test Dummy User, This will be checked again after importing
run: |
ldapsearch -H ldap://localhost:3890 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
- name: Stop LLDAP sqlite
run: pkill lldap
- name: Export and Converting to Postgress
run: |
curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
chmod +x ./helper.sh
./helper.sh | sqlite3 ./users.db > ./dump.sql
sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" -e '1s/^/BEGIN;\n/' -e '$aCOMMIT;' ./dump.sql
- name: Create schema on postgres
run: |
bin/lldap create_schema -d postgres://lldapuser:lldappass@localhost:5432/lldap
- name: Copy converted db to postgress and import
run: |
docker ps -a
docker cp ./dump.sql postgresql:/tmp/dump.sql
docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql"
rm ./dump.sql
- name: Export and Converting to mariadb
run: |
curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
chmod +x ./helper.sh
./helper.sh | sqlite3 ./users.db > ./dump.sql
cp ./dump.sql ./dump-no-sed.sql
sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
- name: Create schema on mariadb
run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3306/lldap
- name: Copy converted db to mariadb and import
run: |
docker ps -a
docker cp ./dump.sql mariadb:/tmp/dump.sql
docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
rm ./dump.sql
- name: Export and Converting to mysql
run: |
curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
chmod +x ./helper.sh
./helper.sh | sqlite3 ./users.db > ./dump.sql
sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
- name: Create schema on mysql
run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3307/lldap
- name: Copy converted db to mysql and import
run: |
docker ps -a
docker cp ./dump.sql mysql:/tmp/dump.sql
docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
rm ./dump.sql
- name: Run lldap with postgres DB and healthcheck again
run: |
bin/lldap run &
sleep 10s
bin/lldap healthcheck
env:
LLDAP_database_url: postgres://lldapuser:lldappass@localhost:5432/lldap
LLDAP_ldap_port: 3891
LLDAP_http_port: 17171
LLDAP_LDAP_USER_PASS: ldappass
LLDAP_JWT_SECRET: somejwtsecret
- name: Run lldap with mariaDB and healthcheck again
run: |
bin/lldap run &
sleep 10s
bin/lldap healthcheck
env:
LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3306/lldap
LLDAP_ldap_port: 3892
LLDAP_http_port: 17172
LLDAP_JWT_SECRET: somejwtsecret
- name: Run lldap with mysql and healthcheck again
run: |
bin/lldap run &
sleep 10s
bin/lldap healthcheck
env:
LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3307/lldap
LLDAP_ldap_port: 3893
LLDAP_http_port: 17173
LLDAP_JWT_SECRET: somejwtsecret
- name: Test Dummy User
run: |
ldapsearch -H ldap://localhost:3891 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
ldapsearch -H ldap://localhost:3892 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
ldapsearch -H ldap://localhost:3893 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
build-docker-image: build-docker-image:
needs: [build-ui, build-bin] needs: [build-ui, build-bin]
@ -356,6 +545,8 @@ jobs:
name: Create release artifacts name: Create release artifacts
if: github.event_name == 'release' if: github.event_name == 'release'
runs-on: ubuntu-latest runs-on: ubuntu-latest
permissions:
contents: write
steps: steps:
- name: Download all artifacts - name: Download all artifacts
uses: actions/download-artifact@v3 uses: actions/download-artifact@v3

View File

@ -65,7 +65,8 @@ a transaction:
``` ```
sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \ sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
-e '1s/^/START TRANSACTION;\n/' \ -e '1s/^/START TRANSACTION;\n/' \
-e '$aCOMMIT;' /path/to/dump.sql -e '$aCOMMIT;' \
-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql
``` ```
### To MariaDB ### To MariaDB
@ -77,7 +78,8 @@ strings. Use the following command to remove those and perform the additional My
sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \ sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \
-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \ -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
-e '1s/^/START TRANSACTION;\n/' \ -e '1s/^/START TRANSACTION;\n/' \
-e '$aCOMMIT;' /path/to/dump.sql -e '$aCOMMIT;' \
-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql
``` ```
## Insert data ## Insert data
@ -103,3 +105,5 @@ or
Modify your `database_url` in `lldap_config.toml` (or `LLDAP_DATABASE_URL` in the env) Modify your `database_url` in `lldap_config.toml` (or `LLDAP_DATABASE_URL` in the env)
to point to your new database (the same value used when generating schema). Restart to point to your new database (the same value used when generating schema). Restart
LLDAP and check the logs to ensure there were no errors. LLDAP and check the logs to ensure there were no errors.
#### More details/examples can be seen in the CI process [here](https://raw.githubusercontent.com/nitnelave/lldap/main/.github/workflows/docker-build-static.yml), look for the job `lldap-database-migration-test`