From c7c6d953346514881df99007952b85a987e1ea82 Mon Sep 17 00:00:00 2001 From: Dedy Martadinata S Date: Tue, 28 Mar 2023 18:59:23 +0700 Subject: [PATCH] docker: Add DB migration tests in the CI --- .github/workflows/docker-build-static.yml | 199 +++++++++++++++++++++- docs/database_migration.md | 10 +- 2 files changed, 202 insertions(+), 7 deletions(-) diff --git a/.github/workflows/docker-build-static.yml b/.github/workflows/docker-build-static.yml index d331127..b7ab660 100644 --- a/.github/workflows/docker-build-static.yml +++ b/.github/workflows/docker-build-static.yml @@ -4,12 +4,18 @@ on: push: branches: - 'main' + paths-ignore: + - 'docs/**' + - 'example_configs/**' release: types: - 'published' pull_request: branches: - 'main' + paths-ignore: + - 'docs/**' + - 'example_configs/**' workflow_dispatch: inputs: msg: @@ -164,9 +170,9 @@ jobs: name: ${{ matrix.target }}-lldap_set_password-bin path: target/${{ matrix.target }}/release/lldap_set_password - lldap-database-integration-test: + lldap-database-init-test: needs: [build-ui,build-bin] - name: LLDAP test + name: LLDAP database init test runs-on: ubuntu-latest services: mariadb: @@ -178,6 +184,7 @@ jobs: MYSQL_PASSWORD: lldappass MYSQL_DATABASE: lldap MYSQL_ROOT_PASSWORD: rootpass + options: --name mariadb postgresql: image: postgres:latest @@ -187,6 +194,7 @@ jobs: POSTGRES_USER: lldapuser POSTGRES_PASSWORD: lldappass POSTGRES_DB: lldap + options: --name postgresql steps: - name: Download artifacts @@ -194,8 +202,7 @@ jobs: with: name: x86_64-unknown-linux-musl-lldap-bin path: bin/ - - name: Where is the bin? - run: ls -alR bin + - name: Set executables to LLDAP run: chmod +x bin/lldap @@ -230,8 +237,190 @@ jobs: LLDAP_database_url: sqlite://users.db?mode=rwc LLDAP_ldap_port: 3892 LLDAP_http_port: 17172 + + - name: Check DB container logs + run: | + docker logs -n 20 mariadb + docker logs -n 20 postgresql + lldap-database-migration-test: + needs: [build-ui,build-bin] + name: LLDAP database migration test + runs-on: ubuntu-latest + services: + postgresql: + image: postgres:latest + ports: + - 5432:5432 + env: + POSTGRES_USER: lldapuser + POSTGRES_PASSWORD: lldappass + POSTGRES_DB: lldap + options: --name postgresql + mariadb: + image: mariadb:latest + ports: + - 3306:3306 + env: + MYSQL_USER: lldapuser + MYSQL_PASSWORD: lldappass + MYSQL_DATABASE: lldap + MYSQL_ROOT_PASSWORD: rootpass + options: --name mariadb + mysql: + image: mysql:latest + ports: + - 3307:3306 + env: + MYSQL_USER: lldapuser + MYSQL_PASSWORD: lldappass + MYSQL_DATABASE: lldap + MYSQL_ROOT_PASSWORD: rootpass + options: --name mysql + + steps: + - name: Download LLDAP artifacts + uses: actions/download-artifact@v3 + with: + name: x86_64-unknown-linux-musl-lldap-bin + path: bin/ + + - name: Download LLDAP set password + uses: actions/download-artifact@v3 + with: + name: x86_64-unknown-linux-musl-lldap_set_password-bin + path: bin/ + + - name: Set executables to LLDAP and LLDAP set password + run: | + chmod +x bin/lldap + chmod +x bin/lldap_set_password + + - name: Install sqlite3 and ldap-utils for exporting and searching dummy user + run: sudo apt update && sudo apt install -y sqlite3 ldap-utils + + - name: Run lldap with sqlite DB and healthcheck + run: | + bin/lldap run & + sleep 10s + bin/lldap healthcheck + env: + LLDAP_database_url: sqlite://users.db?mode=rwc + LLDAP_ldap_port: 3890 + LLDAP_http_port: 17170 + LLDAP_LDAP_USER_PASS: ldappass + LLDAP_JWT_SECRET: somejwtsecret + + - name: Create dummy user + run: | + TOKEN=$(curl -X POST -H "Content-Type: application/json" -d '{"username": "admin", "password": "ldappass"}' http://localhost:17170/auth/simple/login | cut -c 11-277) + echo "$TOKEN" + curl 'http://localhost:17170/api/graphql' -H 'Content-Type: application/json' -H "Authorization: Bearer ${TOKEN//[$'\t\r\n ']}" --data-binary '{"query":"mutation{\n createUser(user:\n {\n id: \"dummyuser\",\n email: \"dummyuser@example.com\"\n }\n )\n {\n id\n email\n }\n}\n\n\n"}' --compressed + bin/lldap_set_password --base-url http://localhost:17170 --admin-username admin --admin-password ldappass --token $TOKEN --username dummyuser --password dummypassword + + - name: Test Dummy User, This will be checked again after importing + run: | + ldapsearch -H ldap://localhost:3890 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com" + + - name: Stop LLDAP sqlite + run: pkill lldap + + - name: Export and Converting to Postgress + run: | + curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh + chmod +x ./helper.sh + ./helper.sh | sqlite3 ./users.db > ./dump.sql + sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" -e '1s/^/BEGIN;\n/' -e '$aCOMMIT;' ./dump.sql + + - name: Create schema on postgres + run: | + bin/lldap create_schema -d postgres://lldapuser:lldappass@localhost:5432/lldap + + - name: Copy converted db to postgress and import + run: | + docker ps -a + docker cp ./dump.sql postgresql:/tmp/dump.sql + docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql" + rm ./dump.sql + + - name: Export and Converting to mariadb + run: | + curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh + chmod +x ./helper.sh + ./helper.sh | sqlite3 ./users.db > ./dump.sql + cp ./dump.sql ./dump-no-sed.sql + sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql + sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql + + - name: Create schema on mariadb + run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3306/lldap + + - name: Copy converted db to mariadb and import + run: | + docker ps -a + docker cp ./dump.sql mariadb:/tmp/dump.sql + docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql" + rm ./dump.sql + + - name: Export and Converting to mysql + run: | + curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh + chmod +x ./helper.sh + ./helper.sh | sqlite3 ./users.db > ./dump.sql + sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql + sed -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql + + - name: Create schema on mysql + run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3307/lldap + + - name: Copy converted db to mysql and import + run: | + docker ps -a + docker cp ./dump.sql mysql:/tmp/dump.sql + docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql" + rm ./dump.sql + + - name: Run lldap with postgres DB and healthcheck again + run: | + bin/lldap run & + sleep 10s + bin/lldap healthcheck + env: + LLDAP_database_url: postgres://lldapuser:lldappass@localhost:5432/lldap + LLDAP_ldap_port: 3891 + LLDAP_http_port: 17171 + LLDAP_LDAP_USER_PASS: ldappass + LLDAP_JWT_SECRET: somejwtsecret + + - name: Run lldap with mariaDB and healthcheck again + run: | + bin/lldap run & + sleep 10s + bin/lldap healthcheck + env: + LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3306/lldap + LLDAP_ldap_port: 3892 + LLDAP_http_port: 17172 + LLDAP_JWT_SECRET: somejwtsecret + + - name: Run lldap with mysql and healthcheck again + run: | + bin/lldap run & + sleep 10s + bin/lldap healthcheck + env: + LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3307/lldap + LLDAP_ldap_port: 3893 + LLDAP_http_port: 17173 + LLDAP_JWT_SECRET: somejwtsecret + + - name: Test Dummy User + run: | + ldapsearch -H ldap://localhost:3891 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com" + ldapsearch -H ldap://localhost:3892 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com" + ldapsearch -H ldap://localhost:3893 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com" + build-docker-image: needs: [build-ui, build-bin] name: Build Docker image @@ -356,6 +545,8 @@ jobs: name: Create release artifacts if: github.event_name == 'release' runs-on: ubuntu-latest + permissions: + contents: write steps: - name: Download all artifacts uses: actions/download-artifact@v3 diff --git a/docs/database_migration.md b/docs/database_migration.md index cf29b81..5ad5954 100644 --- a/docs/database_migration.md +++ b/docs/database_migration.md @@ -65,7 +65,8 @@ a transaction: ``` sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \ -e '1s/^/START TRANSACTION;\n/' \ --e '$aCOMMIT;' /path/to/dump.sql +-e '$aCOMMIT;' \ +-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql ``` ### To MariaDB @@ -77,7 +78,8 @@ strings. Use the following command to remove those and perform the additional My sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \ -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \ -e '1s/^/START TRANSACTION;\n/' \ --e '$aCOMMIT;' /path/to/dump.sql +-e '$aCOMMIT;' \ +-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql ``` ## Insert data @@ -102,4 +104,6 @@ or Modify your `database_url` in `lldap_config.toml` (or `LLDAP_DATABASE_URL` in the env) to point to your new database (the same value used when generating schema). Restart -LLDAP and check the logs to ensure there were no errors. \ No newline at end of file +LLDAP and check the logs to ensure there were no errors. + +#### More details/examples can be seen in the CI process [here](https://raw.githubusercontent.com/nitnelave/lldap/main/.github/workflows/docker-build-static.yml), look for the job `lldap-database-migration-test`