docker: Add DB migration tests in the CI

.github/workflows/docker-build-static.yml

@@ -4,12 +4,18 @@ on:
   push:
     branches:
       - 'main'
+    paths-ignore:
+      - 'docs/**'
+      - 'example_configs/**'
   release:
     types:
       - 'published'
   pull_request:
     branches:
       - 'main'
+    paths-ignore:
+      - 'docs/**'
+      - 'example_configs/**'
   workflow_dispatch:
     inputs:
       msg:
@@ -164,9 +170,9 @@ jobs:
           name: ${{ matrix.target }}-lldap_set_password-bin
           path: target/${{ matrix.target }}/release/lldap_set_password
 
-  lldap-database-integration-test:
+  lldap-database-init-test:
     needs: [build-ui,build-bin]
-    name: LLDAP test
+    name: LLDAP database init test
     runs-on: ubuntu-latest
     services:
         mariadb:
@@ -178,6 +184,7 @@ jobs:
             MYSQL_PASSWORD: lldappass
             MYSQL_DATABASE: lldap
             MYSQL_ROOT_PASSWORD: rootpass
+          options: --name mariadb
 
         postgresql:
            image: postgres:latest
@@ -187,6 +194,7 @@ jobs:
              POSTGRES_USER: lldapuser
              POSTGRES_PASSWORD: lldappass
              POSTGRES_DB: lldap
+           options: --name postgresql
 
     steps:
        - name: Download artifacts
@@ -194,8 +202,7 @@ jobs:
          with:
            name: x86_64-unknown-linux-musl-lldap-bin
            path: bin/
-       - name: Where is the bin?
+
-         run: ls -alR bin
        - name: Set executables to LLDAP
          run: chmod +x bin/lldap
 
@@ -231,6 +238,188 @@ jobs:
            LLDAP_ldap_port: 3892
            LLDAP_http_port: 17172
            
+       - name: Check DB container logs
+         run: |
+              docker logs -n 20 mariadb
+              docker logs -n 20 postgresql
+
+  lldap-database-migration-test:
+    needs: [build-ui,build-bin]
+    name: LLDAP database migration test
+    runs-on: ubuntu-latest
+    services:
+        postgresql:
+           image: postgres:latest
+           ports:
+             - 5432:5432
+           env:
+             POSTGRES_USER: lldapuser
+             POSTGRES_PASSWORD: lldappass
+             POSTGRES_DB: lldap
+           options: --name postgresql
+        mariadb:
+          image: mariadb:latest
+          ports:
+            - 3306:3306
+          env:
+            MYSQL_USER: lldapuser
+            MYSQL_PASSWORD: lldappass
+            MYSQL_DATABASE: lldap
+            MYSQL_ROOT_PASSWORD: rootpass
+          options: --name mariadb
+        mysql:
+          image: mysql:latest
+          ports:
+            - 3307:3306
+          env:
+            MYSQL_USER: lldapuser
+            MYSQL_PASSWORD: lldappass
+            MYSQL_DATABASE: lldap
+            MYSQL_ROOT_PASSWORD: rootpass    
+          options: --name mysql
+      
+
+    steps:
+       - name: Download LLDAP artifacts
+         uses: actions/download-artifact@v3
+         with:
+           name: x86_64-unknown-linux-musl-lldap-bin
+           path: bin/
+           
+       - name: Download LLDAP set password
+         uses: actions/download-artifact@v3
+         with:
+           name: x86_64-unknown-linux-musl-lldap_set_password-bin 
+           path: bin/
+
+       - name: Set executables to LLDAP and LLDAP set password
+         run: |
+              chmod +x bin/lldap
+              chmod +x bin/lldap_set_password
+
+       - name: Install sqlite3 and ldap-utils for exporting and searching dummy user
+         run: sudo apt update && sudo apt install -y sqlite3 ldap-utils
+
+       - name: Run lldap with sqlite DB and healthcheck
+         run: |
+              bin/lldap run &
+              sleep 10s
+              bin/lldap healthcheck
+         env:
+           LLDAP_database_url: sqlite://users.db?mode=rwc
+           LLDAP_ldap_port: 3890
+           LLDAP_http_port: 17170
+           LLDAP_LDAP_USER_PASS: ldappass
+           LLDAP_JWT_SECRET: somejwtsecret
+
+       - name: Create dummy user
+         run: |
+              TOKEN=$(curl -X POST -H "Content-Type: application/json" -d '{"username": "admin", "password": "ldappass"}' http://localhost:17170/auth/simple/login | cut -c 11-277)
+              echo "$TOKEN"
+              curl 'http://localhost:17170/api/graphql' -H 'Content-Type: application/json' -H "Authorization: Bearer ${TOKEN//[$'\t\r\n ']}" --data-binary '{"query":"mutation{\n  createUser(user:\n    {\n      id: \"dummyuser\",\n      email: \"dummyuser@example.com\"\n    }\n  )\n  {\n    id\n    email\n  }\n}\n\n\n"}' --compressed
+              bin/lldap_set_password --base-url http://localhost:17170 --admin-username admin --admin-password ldappass --token $TOKEN --username dummyuser --password dummypassword
+
+       - name: Test Dummy User, This will be checked again after importing
+         run: |
+              ldapsearch -H ldap://localhost:3890 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
+
+       - name: Stop LLDAP sqlite
+         run: pkill lldap
+
+       - name: Export and Converting to Postgress
+         run: |
+              curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
+              chmod +x ./helper.sh
+              ./helper.sh | sqlite3 ./users.db > ./dump.sql
+              sed -i -r -e "s/X'([[:xdigit:]]+'[^'])/'\\\x\\1/g" -e '1s/^/BEGIN;\n/' -e '$aCOMMIT;' ./dump.sql
+
+       - name: Create schema on postgres
+         run: |
+              bin/lldap create_schema -d postgres://lldapuser:lldappass@localhost:5432/lldap
+
+       - name: Copy converted db to postgress and import
+         run: |
+              docker ps -a
+              docker cp ./dump.sql postgresql:/tmp/dump.sql
+              docker exec postgresql bash -c "psql -U lldapuser -d lldap < /tmp/dump.sql"
+              rm ./dump.sql
+              
+       - name: Export and Converting to mariadb
+         run: |
+              curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
+              chmod +x ./helper.sh
+              ./helper.sh | sqlite3 ./users.db > ./dump.sql
+              cp ./dump.sql ./dump-no-sed.sql
+              sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \-e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
+              sed  -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
+              
+       - name: Create schema on mariadb
+         run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3306/lldap
+
+       - name: Copy converted db to mariadb and import
+         run: |
+              docker ps -a
+              docker cp ./dump.sql mariadb:/tmp/dump.sql
+              docker exec mariadb bash -c "mariadb -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
+              rm ./dump.sql
+              
+       - name: Export and Converting to mysql
+         run: |
+              curl -L https://raw.githubusercontent.com/nitnelave/lldap/main/scripts/sqlite_dump_commands.sh -o helper.sh
+              chmod +x ./helper.sh
+              ./helper.sh | sqlite3 ./users.db > ./dump.sql
+              sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' -e '1s/^/START TRANSACTION;\n/' -e '$aCOMMIT;' ./dump.sql
+              sed  -i '1 i\SET FOREIGN_KEY_CHECKS = 0;' ./dump.sql
+             
+       - name: Create schema on mysql
+         run: bin/lldap create_schema -d mysql://lldapuser:lldappass@localhost:3307/lldap
+
+       - name: Copy converted db to mysql and import
+         run: |
+              docker ps -a
+              docker cp ./dump.sql mysql:/tmp/dump.sql
+              docker exec mysql bash -c "mysql -ulldapuser -plldappass -f lldap < /tmp/dump.sql"
+              rm ./dump.sql
+           
+       - name: Run lldap with postgres DB and healthcheck again
+         run: |
+              bin/lldap run &
+              sleep 10s
+              bin/lldap healthcheck
+         env:
+           LLDAP_database_url: postgres://lldapuser:lldappass@localhost:5432/lldap
+           LLDAP_ldap_port: 3891
+           LLDAP_http_port: 17171
+           LLDAP_LDAP_USER_PASS: ldappass
+           LLDAP_JWT_SECRET: somejwtsecret
+           
+       - name: Run lldap with mariaDB and healthcheck again
+         run: |
+              bin/lldap run &
+              sleep 10s
+              bin/lldap healthcheck
+         env:
+           LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3306/lldap
+           LLDAP_ldap_port: 3892
+           LLDAP_http_port: 17172
+           LLDAP_JWT_SECRET: somejwtsecret
+           
+       - name: Run lldap with mysql and healthcheck again
+         run: |
+              bin/lldap run &
+              sleep 10s
+              bin/lldap healthcheck
+         env:
+           LLDAP_database_url: mysql://lldapuser:lldappass@localhost:3307/lldap
+           LLDAP_ldap_port: 3893
+           LLDAP_http_port: 17173
+           LLDAP_JWT_SECRET: somejwtsecret
+           
+       - name: Test Dummy User
+         run: |
+              ldapsearch -H ldap://localhost:3891 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
+              ldapsearch -H ldap://localhost:3892 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
+              ldapsearch -H ldap://localhost:3893 -LLL -D "uid=dummyuser,ou=people,dc=example,dc=com" -w 'dummypassword' -s "One" -b "ou=people,dc=example,dc=com"
          
   build-docker-image:
     needs: [build-ui, build-bin]
@@ -356,6 +545,8 @@ jobs:
      name: Create release artifacts
      if: github.event_name == 'release'
      runs-on: ubuntu-latest
+     permissions:
+       contents: write
      steps:
       - name: Download all artifacts
         uses: actions/download-artifact@v3

docs/database_migration.md

@@ -65,7 +65,8 @@ a transaction:
 ```
 sed -i -r -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
 -e '1s/^/START TRANSACTION;\n/' \
--e '$aCOMMIT;' /path/to/dump.sql
+-e '$aCOMMIT;' \
+-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql
 ```
 
 ### To MariaDB
@@ -77,7 +78,8 @@ strings. Use the following command to remove those and perform the additional My
 sed -i -r -e "s/([^']'[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}\.[0-9]{9})\+00:00'([^'])/\1'\2/g" \
 -e 's/^INSERT INTO "?([a-zA-Z0-9_]+)"?/INSERT INTO `\1`/' \
 -e '1s/^/START TRANSACTION;\n/' \
--e '$aCOMMIT;' /path/to/dump.sql
+-e '$aCOMMIT;' \
+-e '1 i\SET FOREIGN_KEY_CHECKS = 0;' /path/to/dump.sql
 ```
 
 ## Insert data
@@ -103,3 +105,5 @@ or
 Modify your `database_url` in `lldap_config.toml` (or `LLDAP_DATABASE_URL` in the env)
 to point to your new database (the same value used when generating schema). Restart
 LLDAP and check the logs to ensure there were no errors.
+
+#### More details/examples can be seen in the CI process [here](https://raw.githubusercontent.com/nitnelave/lldap/main/.github/workflows/docker-build-static.yml), look for the job `lldap-database-migration-test`