Merge pull request #494 from databasus/develop #722
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI and Release | |
| on: | |
| push: | |
| branches: ["**"] | |
| pull_request: | |
| branches: ["**"] | |
| workflow_dispatch: | |
| jobs: | |
| lint-backend: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: self-hosted | |
| container: | |
| image: golang:1.26.1 | |
| volumes: | |
| - /runner-cache/go-pkg:/go/pkg/mod | |
| - /runner-cache/go-build:/root/.cache/go-build | |
| - /runner-cache/golangci-lint:/root/.cache/golangci-lint | |
| - /runner-cache/apt-archives:/var/cache/apt/archives | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Configure Git for container | |
| run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Download Go modules | |
| run: | | |
| cd backend | |
| go mod download | |
| - name: Install golangci-lint | |
| run: | | |
| curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3 | |
| echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
| - name: Install swag for swagger generation | |
| run: go install github.com/swaggo/swag/cmd/swag@v1.16.4 | |
| - name: Generate swagger docs | |
| run: | | |
| 8000 | cd backend |
| swag init -d . -g cmd/main.go -o swagger | |
| - name: Run golangci-lint | |
| run: | | |
| cd backend | |
| golangci-lint run | |
| - name: Verify go mod tidy | |
| run: | | |
| cd backend | |
| go mod tidy | |
| git diff --exit-code go.mod go.sum || (echo "go mod tidy made changes, please run 'go mod tidy' and commit the changes" && exit 1) | |
| lint-frontend: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Install dependencies | |
| run: | | |
| cd frontend | |
| npm ci | |
| - name: Check if prettier was run | |
| run: | | |
| cd frontend | |
| npm run format | |
| git diff --exit-code || (echo "Prettier made changes, please run 'npm run format' and commit the changes" && exit 1) | |
| - name: Check if linter was run | |
| run: | | |
| cd frontend | |
| npm run lint | |
| - name: Build frontend | |
| run: | | |
| cd frontend | |
| npm run build | |
| lint-agent: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.26.1" | |
| cache-dependency-path: agent/go.sum | |
| - name: Download Go modules | |
| run: | | |
| cd agent | |
| go mod download | |
| - name: Install golangci-lint | |
| run: | | |
| curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.11.3 | |
| echo "$(go env GOPATH)/bin" >> $GITHUB_PATH | |
| - name: Run golangci-lint | |
| run: | | |
| cd agent | |
| golangci-lint run | |
| - name: Verify go mod tidy | |
| run: | | |
| cd agent | |
| go mod tidy | |
| git diff --exit-code go.mod go.sum || (echo "go mod tidy made changes, please run 'go mod tidy' and commit the changes" && exit 1) | |
| test-frontend: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: ubuntu-latest | |
| needs: [lint-frontend] | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: "20" | |
| - name: Install dependencies | |
| run: | | |
| cd frontend | |
| npm ci | |
| - name: Run frontend tests | |
| run: | | |
| cd frontend | |
| npm run test | |
| test-agent: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: ubuntu-latest | |
| needs: [lint-agent] | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: "1.26.1" | |
| cache-dependency-path: agent/go.sum | |
| - name: Download Go modules | |
| run: | | |
| cd agent | |
| go mod download | |
| - name: Run Go tests | |
| run: | | |
| cd agent | |
| go test -count=1 -failfast ./internal/... | |
| e2e-agent: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: ubuntu-latest | |
| needs: [lint-agent] | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Run e2e tests | |
| run: | | |
| cd agent | |
| make e2e | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| cd agent/e2e | |
| docker compose down -v --rmi local || true | |
| rm -rf artifacts || true | |
| e2e-agent-backup-restore: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: ubuntu-latest | |
| needs: [lint-agent] | |
| strategy: | |
| matrix: | |
| pg_version: [15, 16, 17, 18] | |
| fail-fast: false | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Run backup-restore e2e (PG ${{ matrix.pg_version }}) | |
| run: | | |
| cd agent | |
| make e2e-backup-restore PG_VERSION=${{ matrix.pg_version }} | |
| - name: Cleanup | |
| if: always() | |
| run: | | |
| cd agent/e2e | |
| docker compose -f docker-compose.backup-restore.yml down -v --rmi local || true | |
| rm -rf artifacts || true | |
| # Self-hosted: performant high-frequency CPU is used to start many containers and run tests fast. Tests | |
| # step is bottle-neck, because we need a lot of containers and cannot parallelize tests due to shared resources | |
| test-backend: | |
| if: github.ref != 'refs/heads/develop' | |
| runs-on: self-hosted | |
| needs: [lint-backend] | |
| container: | |
| image: golang:1.26.1 | |
| options: --privileged -v /var/run/docker.sock:/var/run/docker.sock --add-host=host.docker.internal:host-gateway | |
| volumes: | |
| - /runner-cache/go-pkg:/go/pkg/mod | |
| - /runner-cache/go-build:/root/.cache/go-build | |
| - /runner-cache/apt-archives:/var/cache/apt/archives | |
| steps: | |
| - name: Install Docker CLI | |
| run: | | |
| apt-get update -qq | |
| apt-get install -y -qq docker.io docker-compose netcat-openbsd wget | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Configure Git for container | |
| run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Download Go modules | |
| run: | | |
| cd backend | |
| go mod download | |
| - name: Create .env file for testing | |
| run: | | |
| cd backend | |
| cat > .env << EOF | |
| # docker-compose.yml | |
| DEV_DB_NAME=databasus | |
| DEV_DB_USERNAME=postgres | |
| DEV_DB_PASSWORD=Q1234567 | |
| #app | |
| ENV_MODE=development | |
| # db - using 172.17.0.1 to access host from container | |
| DATABASE_DSN=host=172.17.0.1 user=postgres password=Q1234567 dbname=databasus port=5437 sslmode=disable | |
| DATABASE_URL=postgres://postgres:Q1234567@172.17.0.1:5437/databasus?sslmode=disable | |
| # migrations | |
| GOOSE_DRIVER=postgres | |
| GOOSE_DBSTRING=postgres://postgres:Q1234567@172.17.0.1:5437/databasus?sslmode=disable | |
| GOOSE_MIGRATION_DIR=./migrations | |
| # testing | |
| TEST_LOCALHOST=172.17.0.1 | |
| IS_SKIP_EXTERNAL_RESOURCES_TESTS=true | |
| # to get Google Drive env variables: add storage in UI and copy data from added storage here | |
| TEST_GOOGLE_DRIVE_CLIENT_ID=${{ secrets.TEST_GOOGLE_DRIVE_CLIENT_ID }} | |
| TEST_GOOGLE_DRIVE_CLIENT_SECRET=${{ secrets.TEST_GOOGLE_DRIVE_CLIENT_SECRET }} | |
| TEST_GOOGLE_DRIVE_TOKEN_JSON=${{ secrets.TEST_GOOGLE_DRIVE_TOKEN_JSON }} | |
| # testing DBs | |
| TEST_POSTGRES_12_PORT=5000 | |
| TEST_POSTGRES_13_PORT=5001 | |
| TEST_POSTGRES_14_PORT=5002 | |
| TEST_POSTGRES_15_PORT=5003 | |
| TEST_POSTGRES_16_PORT=5004 | |
| TEST_POSTGRES_17_PORT=5005 | |
| TEST_POSTGRES_18_PORT=5006 | |
| # testing S3 | |
| TEST_MINIO_PORT=9000 | |
| TEST_MINIO_CONSOLE_PORT=9001 | |
| # testing Azure Blob | |
| TEST_AZURITE_BLOB_PORT=10000 | |
| # testing NAS | |
| TEST_NAS_PORT=7006 | |
| # testing FTP | |
| TEST_FTP_PORT=7007 | |
| # testing SFTP | |
| TEST_SFTP_PORT=7008 | |
| # testing MySQL | |
| TEST_MYSQL_57_PORT=33057 | |
| TEST_MYSQL_80_PORT=33080 | |
| TEST_MYSQL_84_PORT=33084 | |
| TEST_MYSQL_90_PORT=33090 | |
| # testing MariaDB | |
| TEST_MARIADB_55_PORT=33055 | |
| TEST_MARIADB_101_PORT=33101 | |
| TEST_MARIADB_102_PORT=33102 | |
| TEST_MARIADB_103_PORT=33103 | |
| TEST_MARIADB_104_PORT=33104 | |
| TEST_MARIADB_105_PORT=33105 | |
| TEST_MARIADB_106_PORT=33106 | |
| TEST_MARIADB_1011_PORT=33111 | |
| TEST_MARIADB_114_PORT=33114 | |
| TEST_MARIADB_118_PORT=33118 | |
| TEST_MARIADB_120_PORT=33120 | |
| # supabase | |
| TEST_SUPABASE_HOST=${{ secrets.TEST_SUPABASE_HOST }} | |
| TEST_SUPABASE_PORT=${{ secrets.TEST_SUPABASE_PORT }} | |
| TEST_SUPABASE_USERNAME=${{ secrets.TEST_SUPABASE_USERNAME }} | |
| TEST_SUPABASE_PASSWORD=${{ secrets.TEST_SUPABASE_PASSWORD }} | |
| TEST_SUPABASE_DATABASE=${{ secrets.TEST_SUPABASE_DATABASE }} | |
| # testing MongoDB | |
| TEST_MONGODB_40_PORT=27040 | |
| TEST_MONGODB_42_PORT=27042 | |
| TEST_MONGODB_44_PORT=27044 | |
| TEST_MONGODB_50_PORT=27050 | |
| TEST_MONGODB_60_PORT=27060 | |
| TEST_MONGODB_70_PORT=27070 | |
| TEST_MONGODB_82_PORT=27082 | |
| # Valkey (cache) - using 172.17.0.1 | |
| VALKEY_HOST=172.17.0.1 | |
| VALKEY_PORT=6379 | |
| VALKEY_USERNAME= | |
| VALKEY_PASSWORD= | |
| VALKEY_IS_SSL=false | |
| # Host for test databases (container -> host) | |
| TEST_DB_HOST=172.17.0.1 | |
| EOF | |
| - name: Start test containers | |
| run: | | |
| cd backend | |
| docker compose -f docker-compose.yml.example up -d | |
| - name: Wait for containers to be ready | |
| run: | | |
| # Wait for main dev database | |
| timeout 60 bash -c 'until docker exec dev-db pg_isready -h localhost -p 5437 -U postgres; do sleep 2; done' | |
| # Wait for Valkey (cache) | |
| echo "Waiting for Valkey..." | |
| timeout 60 bash -c 'until docker exec dev-valkey valkey-cli ping 2>/dev/null | grep -q PONG; do sleep 2; done' | |
| echo "Valkey is ready!" | |
| # Wait for test databases (using 172.17.0.1 from container) | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 5000; do sleep 2; done' | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 5001; do sleep 2; done' | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 5002; do sleep 2; done' | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 5003; do sleep 2; done' | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 5004; do sleep 2; done' | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 5005; do sleep 2; done' | |
| # Wait for MinIO | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 9000; do sleep 2; done' | |
| # Wait for Azurite | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 10000; do sleep 2; done' | |
| # Wait for FTP | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 7007; do sleep 2; done' | |
| # Wait for SFTP | |
| timeout 60 bash -c 'until nc -z 172.17.0.1 7008; do sleep 2; done' | |
| # Wait for MySQL containers | |
| echo "Waiting for MySQL 5.7..." | |
| timeout 120 bash -c 'until docker exec test-mysql-57 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MySQL 8.0..." | |
| timeout 120 bash -c 'until docker exec test-mysql-80 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MySQL 8.4..." | |
| timeout 120 bash -c 'until docker exec test-mysql-84 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MySQL 9.0..." | |
| timeout 120 bash -c 'until docker exec test-mysql-90 mysqladmin ping -h localhost -u root -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| # Wait for MariaDB containers | |
| echo "Waiting for MariaDB 5.5..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-55 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 10.1..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-101 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 10.2..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-102 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 10.3..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-103 mysqladmin ping -h localhost -prootpassword --silent 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 10.4..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-104 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 10.5..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-105 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 10.6..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-106 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 10.11..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-1011 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 11.4..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-114 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 11.8..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-118 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MariaDB 12.0..." | |
| timeout 120 bash -c 'until docker exec test-mariadb-120 healthcheck.sh --connect --innodb_initialized 2>/dev/null; do sleep 2; done' | |
| # Wait for MongoDB containers | |
| echo "Waiting for MongoDB 4.0..." | |
| timeout 120 bash -c 'until docker exec test-mongodb-40 mongo --eval "db.adminCommand(\"ping\")" -u root -p rootpassword --authenticationDatabase admin 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MongoDB 4.2..." | |
| timeout 120 bash -c 'until docker exec test-mongodb-42 mongo --eval "db.adminCommand(\"ping\")" -u root -p rootpassword --authenticationDatabase admin 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MongoDB 4.4..." | |
| timeout 120 bash -c 'until docker exec test-mongodb-44 mongo --eval "db.adminCommand(\"ping\")" -u root -p rootpassword --authenticationDatabase admin 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MongoDB 5.0..." | |
| timeout 120 bash -c 'until docker exec test-mongodb-50 mongosh --eval "db.adminCommand(\"ping\")" -u root -p rootpassword --authenticationDatabase admin 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MongoDB 6.0..." | |
| timeout 120 bash -c 'until docker exec test-mongodb-60 mongosh --eval "db.adminCommand(\"ping\")" -u root -p rootpassword --authenticationDatabase admin 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MongoDB 7.0..." | |
| timeout 120 bash -c 'until docker exec test-mongodb-70 mongosh --eval "db.adminCommand(\"ping\")" -u root -p rootpassword --authenticationDatabase admin 2>/dev/null; do sleep 2; done' | |
| echo "Waiting for MongoDB 8.2..." | |
| timeout 120 bash -c 'until docker exec test-mongodb-82 mongosh --eval "db.adminCommand(\"ping\")" -u root -p rootpassword --authenticationDatabase admin 2>/dev/null; do sleep 2; done' | |
| - name: Create data and temp directories | |
| run: | | |
| # Create directories that are used for backups and restore | |
| # These paths match what's configured in config.go | |
| mkdir -p databasus-data/backups | |
| mkdir -p databasus-data/temp | |
| - name: Install database client dependencies | |
| run: | | |
| apt-get update -qq | |
| apt-get install -y -qq libncurses6 libpq5 | |
| ln -sf /usr/lib/x86_64-linux-gnu/libncurses.so.6 /usr/lib/x86_64-linux-gnu/libncurses.so.5 || true | |
| ln -sf /usr/lib/x86_64-linux-gnu/libtinfo.so.6 /usr/lib/x86_64-linux-gnu/libtinfo.so.5 || true | |
| - name: Setup PostgreSQL, MySQL and MariaDB client tools from pre-built assets | |
| run: | | |
| cd backend/tools | |
| # Create directory structure | |
| mkdir -p postgresql mysql mariadb mongodb/bin | |
| # Copy PostgreSQL client tools (12-18) from pre-built assets | |
| for version in 12 13 14 15 16 17 18; do | |
| mkdir -p postgresql/postgresql-$version | |
| cp -r ../../assets/tools/x64/postgresql/postgresql-$version/bin postgresql/postgresql-$version/ | |
| done | |
| # Copy MySQL client tools (5.7, 8.0, 8.4, 9) from pre-built assets | |
| for version in 5.7 8.0 8.4 9; do | |
| mkdir -p mysql/mysql-$version | |
| cp -r ../../assets/tools/x64/mysql/mysql-$version/bin mysql/mysql-$version/ | |
| done | |
| # Copy MariaDB client tools (10.6, 12.1) from pre-built assets | |
| for version in 10.6 12.1; do | |
| mkdir -p mariadb/mariadb-$version | |
| cp -r ../../assets/tools/x64/mariadb/mariadb-$version/bin mariadb/mariadb-$version/ | |
| done | |
| # Make all binaries executable | |
| chmod +x postgresql/*/bin/* | |
| chmod +x mysql/*/bin/* | |
| chmod +x mariadb/*/bin/* | |
| echo "Pre-built client tools setup complete" | |
| - name: Install MongoDB Database Tools | |
| run: | | |
| cd backend/tools | |
| # MongoDB Database Tools must be downloaded (not in pre-built assets) | |
| # They are backward compatible - single version supports all servers (4.0-8.0) | |
| MONGODB_TOOLS_URL="https://fastdl.mongodb.org/tools/db/mongodb-database-tools-debian12-x86_64-100.10.0.deb" | |
| echo "Downloading MongoDB Database Tools..." | |
| wget -q "$MONGODB_TOOLS_URL" -O /tmp/mongodb-database-tools.deb | |
| echo "Installing MongoDB Database Tools..." | |
| dpkg -i /tmp/mongodb-database-tools.deb || apt-get install -f -y --no-install-recommends | |
| # Create symlinks to tools directory | |
| ln -sf /usr/bin/mongodump mongodb/bin/mongodump | |
| ln -sf /usr/bin/mongorestore mongodb/bin/mongorestore | |
| rm -f /tmp/mongodb-database-tools.deb | |
| echo "MongoDB Database Tools installed successfully" | |
| - name: Verify MariaDB client tools exist | |
| run: | | |
| cd backend/tools | |
| echo "Checking MariaDB client tools..." | |
| if [ -f "mariadb/mariadb-10.6/bin/mariadb-dump" ]; then | |
| echo "MariaDB 10.6 client tools found" | |
| ls -la mariadb/mariadb-10.6/bin/ | |
| else | |
| echo "MariaDB 10.6 client tools NOT found" | |
| fi | |
| if [ -f "mariadb/mariadb-12.1/bin/mariadb-dump" ]; then | |
| echo "MariaDB 12.1 client tools found" | |
| ls -la mariadb/mariadb-12.1/bin/ | |
| else | |
| echo "MariaDB 12.1 client tools NOT found" | |
| fi | |
| - name: Verify MongoDB Database Tools exist | |
| run: | | |
| cd backend/tools | |
| echo "Checking MongoDB Database Tools..." | |
| if [ -f "mongodb/bin/mongodump" ]; then | |
| echo "MongoDB Database Tools found" | |
| ls -la mongodb/bin/ | |
| mongodb/bin/mongodump --version || true | |
| else | |
| echo "MongoDB Database Tools NOT found" | |
| fi | |
| - name: Run database migrations | |
| run: | | |
| cd backend | |
| go install github.com/pressly/goose/v3/cmd/goose@v3.24.3 | |
| goose up | |
| - name: Run Go tests | |
| run: | | |
| cd backend | |
| go test -p=1 -count=1 -failfast -timeout 10m ./internal/... | |
| - name: Stop test containers | |
| if: always() | |
| run: | | |
| cd backend | |
| # Stop and remove containers (keeping images for next run) | |
| docker compose -f docker-compose.yml.example down -v | |
| # Clean up all data directories created by docker-compose | |
| echo "Cleaning up data directories..." | |
| rm -rf pgdata || true | |
| rm -rf valkey-data || true | |
| rm -rf mysqldata || true | |
| rm -rf mariadbdata || true | |
| rm -rf temp/nas || true | |
| rm -rf databasus-data || true | |
| # Also clean root-level databasus-data if exists | |
| cd .. | |
| rm -rf databasus-data || true | |
| echo "Cleanup complete" | |
| build-and-push-dev: | |
| runs-on: self-hosted | |
| if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/develop' }} | |
| steps: | |
| - name: Clean workspace | |
| run: | | |
| sudo rm -rf "$GITHUB_WORKSPACE"/* || true | |
| sudo rm -rf "$GITHUB_WORKSPACE"/.* || true | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU (enables multi-arch emulation) | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and push dev image | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| push: true | |
| platforms: linux/amd64,linux/arm64 | |
| build-args: | | |
| APP_VERSION=dev-${{ github.sha }} | |
| tags: | | |
| databasus/databasus-dev:latest | |
| databasus/databasus-dev:${{ github.sha }} | |
| determine-version: | |
| runs-on: self-hosted | |
| container: | |
| image: node:20 | |
| needs: [test-backend, test-frontend, test-agent, e2e-agent, e2e-agent-backup-restore] | |
| if: ${{ github.ref == 'refs/heads/main' && !contains(github.event.head_commit.message, '[skip-release]') }} | |
| outputs: | |
| should_release: ${{ steps.version_bump.outputs.should_release }} | |
| new_version: ${{ steps.version_bump.outputs.new_version }} | |
| bump_type: ${{ steps.version_bump.outputs.bump_type }} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| - name: Configure Git for container | |
| run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Install semver | |
| run: npm install -g semver | |
| - name: Get current version | |
| id: current_version | |
| run: | | |
| LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") | |
| echo "current_version=${LATEST_TAG#v}" >> $GITHUB_OUTPUT | |
| echo "Current version: ${LATEST_TAG#v}" | |
| - name: Analyze commits and determine version bump | |
| id: version_bump | |
| shell: bash | |
| run: | | |
| CURRENT_VERSION="${{ steps.current_version.outputs.current_version }}" | |
| LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") | |
| # Get commits since last tag | |
| if [ "$LATEST_TAG" = "v0.0.0" ]; then | |
| COMMITS=$(git log --pretty=format:"%s" --no-merges) | |
| else | |
| COMMITS=$(git log ${LATEST_TAG}..HEAD --pretty=format:"%s" --no-merges) | |
| fi | |
| echo "Analyzing commits:" | |
| echo "$COMMITS" | |
| # Initialize flags | |
| HAS_FEATURE=false | |
| HAS_FIX=false | |
| HAS_BREAKING=false | |
| # Analyze each commit - USE PROCESS SUBSTITUTION to avoid subshell variable scope issues | |
| while IFS= read -r commit; do | |
| if [[ "$commit" =~ ^FEATURE ]]; then | |
| HAS_FEATURE=true | |
| echo "Found FEATURE commit: $commit" | |
| elif [[ "$commit" =~ ^FIX ]]; then | |
| HAS_FIX=true | |
| echo "Found FIX commit: $commit" | |
| elif [[ "$commit" =~ ^REFACTOR ]]; then | |
| HAS_FIX=true # Treat refactor as patch | |
| echo "Found REFACTOR commit: $commit" | |
| fi | |
| # Check for breaking changes | |
| if [[ "$commit" =~ BREAKING[[:space:]]CHANGE ]] || [[ "$commit" =~ "!" ]]; then | |
| HAS_BREAKING=true | |
| echo "Found BREAKING CHANGE: $commit" | |
| fi | |
| done < <(printf '%s\n' "$COMMITS") | |
| # Determine version bump | |
| if [ "$HAS_BREAKING" = true ]; then | |
| BUMP_TYPE="major" | |
| elif [ "$HAS_FEATURE" = true ]; then | |
| BUMP_TYPE="minor" | |
| elif [ "$HAS_FIX" = true ]; then | |
| BUMP_TYPE="patch" | |
| else | |
| BUMP_TYPE="none" | |
| fi | |
| echo "bump_type=$BUMP_TYPE" >> $GITHUB_OUTPUT | |
| if [ "$BUMP_TYPE" != "none" ]; then | |
| NEW_VERSION=$(npx semver -i $BUMP_TYPE $CURRENT_VERSION) | |
| echo "new_version=$NEW_VERSION" >> $GITHUB_OUTPUT | |
| echo "should_release=true" >> $GITHUB_OUTPUT | |
| echo "New version will be: $NEW_VERSION" | |
| else | |
| echo "should_release=false" >> $GITHUB_OUTPUT | |
| echo "No version bump needed" | |
| fi | |
| build-and-push: | |
| runs-on: self-hosted | |
| needs: [determine-version] | |
| if: ${{ needs.determine-version.outputs.should_release == 'true' }} | |
| permissions: | |
| contents: write | |
| steps: | |
| - name: Clean workspace | |
| run: | | |
| sudo rm -rf "$GITHUB_WORKSPACE"/* || true | |
| sudo rm -rf "$GITHUB_WORKSPACE"/.* || true | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Set up QEMU (enables multi-arch emulation) | |
| uses: docker/setup-qemu-action@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v3 | |
| - name: Log in to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USERNAME }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Build and push with version tags | |
| uses: docker/build-push-action@v5 | |
| with: | |
| context: . | |
| push: true | |
| platforms: linux/amd64,linux/arm64 | |
| build-args: | | |
| APP_VERSION=${{ needs.determine-version.outputs.new_version }} | |
| tags: | | |
| databasus/databasus:latest | |
| databasus/databasus:v${{ needs.determine-version.outputs.new_version }} | |
| databasus/databasus:${{ github.sha }} | |
| release: | |
| runs-on: self-hosted | |
| container: | |
| image: node:20 | |
| needs: [determine-version, build-and-push] | |
| if: ${{ needs.determine-version.outputs.should_release == 'true' }} | |
| permissions: | |
| contents: write | |
| pull-requests: write | |
| steps: | |
| - name: Clean workspace | |
| run: | | |
| rm -rf "$GITHUB_WORKSPACE"/* || true | |
| rm -rf "$GITHUB_WORKSPACE"/.* || true | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Configure Git for container | |
| run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Generate changelog | |
| id: changelog | |
| shell: bash | |
| run: | | |
| NEW_VERSION="${{ needs.determine-version.outputs.new_version }}" | |
| LATEST_TAG=$(git describe --tags --abbrev=0 2>/dev/null || echo "v0.0.0") | |
| # Get commits since last tag | |
| if [ "$LATEST_TAG" = "v0.0.0" ]; then | |
| COMMITS=$(git log --pretty=format:"%s|%H|%an|%ad" --date=short --no-merges) | |
| else | |
| COMMITS=$(git log ${LATEST_TAG}..HEAD --pretty=format:"%s|%H|%an|%ad" --date=short --no-merges) | |
| fi | |
| # Create changelog | |
| CHANGELOG="# Changelog\n\n## [${NEW_VERSION}] - $(date +%Y-%m-%d)\n\n" | |
| # Group commits by type and area | |
| FEATURES="" | |
| FIXES="" | |
| REFACTORS="" | |
| # USE PROCESS SUBSTITUTION to avoid subshell variable scope issues | |
| while IFS= read -r line; do | |
| if [ -n "$line" ]; then | |
| COMMIT_MSG=$(echo "$line" | cut -d'|' -f1) | |
| COMMIT_HASH=$(echo "$line" | cut -d'|' -f2) | |
| SHORT_HASH=${COMMIT_HASH:0:7} | |
| # Parse commit message format: TYPE (area): description | |
| if [[ "$COMMIT_MSG" == FEATURE* ]]; then | |
| TEMP="${COMMIT_MSG#FEATURE}" | |
| TEMP="${TEMP#"${TEMP%%[![:space:]]*}"}" | |
| if [[ "$TEMP" == \(* ]]; then | |
| AREA=$(echo "$TEMP" | sed 's/^(\([^)]*\)).*/\1/') | |
| DESC=$(echo "$TEMP" | sed 's/^([^)]*):[[:space:]]*//') | |
| FEATURES="${FEATURES}- **${AREA}**: ${DESC} ([${SHORT_HASH}](https://github.com/${{ github.repository }}/commit/${COMMIT_HASH}))\n" | |
| fi | |
| elif [[ "$COMMIT_MSG" == FIX* ]]; then | |
| TEMP="${COMMIT_MSG#FIX}" | |
| TEMP="${TEMP#"${TEMP%%[![:space:]]*}"}" | |
| if [[ "$TEMP" == \(* ]]; then | |
| AREA=$(echo "$TEMP" | sed 's/^(\([^)]*\)).*/\1/') | |
| DESC=$(echo "$TEMP" | sed 's/^([^)]*):[[:space:]]*//') | |
| FIXES="${FIXES}- **${AREA}**: ${DESC} ([${SHORT_HASH}](https://github.com/${{ github.repository }}/commit/${COMMIT_HASH}))\n" | |
| fi | |
| elif [[ "$COMMIT_MSG" == REFACTOR* ]]; then | |
| TEMP="${COMMIT_MSG#REFACTOR}" | |
| TEMP="${TEMP#"${TEMP%%[![:space:]]*}"}" | |
| if [[ "$TEMP" == \(* ]]; then | |
| AREA=$(echo "$TEMP" | sed 's/^(\([^)]*\)).*/\1/') | |
| DESC=$(echo "$TEMP" | sed 's/^([^)]*):[[:space:]]*//') | |
| REFACTORS="${REFACTORS}- **${AREA}**: ${DESC} ([${SHORT_HASH}](https://github.com/${{ github.repository }}/commit/${COMMIT_HASH}))\n" | |
| fi | |
| fi | |
| fi | |
| done < <(printf '%s\n' "$COMMITS") | |
| # Build changelog sections | |
| if [ -n "$FEATURES" ]; then | |
| CHANGELOG="${CHANGELOG}### ✨ Features\n${FEATURES}\n" | |
| fi | |
| if [ -n "$FIXES" ]; then | |
| CHANGELOG="${CHANGELOG}### 🐛 Bug Fixes\n${FIXES}\n" | |
| fi | |
| if [ -n "$REFACTORS" ]; then | |
| CHANGELOG="${CHANGELOG}### 🔨 Refactoring\n${REFACTORS}\n" | |
| fi | |
| # Add Docker image info | |
| CHANGELOG="${CHANGELOG}### 🐳 Docker\n" | |
| CHANGELOG="${CHANGELOG}- **Image**: \`databasus/databasus:v${NEW_VERSION}\`\n" | |
| CHANGELOG="${CHANGELOG}- **Platforms**: linux/amd64, linux/arm64\n\n" | |
| # Set output for GitHub release | |
| { | |
| echo 'changelog<<EOF' | |
| echo -e "$CHANGELOG" | |
| echo EOF | |
| } >> $GITHUB_OUTPUT | |
| - name: Create GitHub Release | |
| uses: actions/create-release@v1 | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| with: | |
| tag_name: v${{ needs.determine-version.outputs.new_version }} | |
| release_name: Release v${{ needs.determine-version.outputs.new_version }} | |
| body: ${{ steps.changelog.outputs.changelog }} | |
| draft: false | |
| prerelease: false | |
| publish-helm-chart: | |
| runs-on: self-hosted | |
| container: | |
| image: alpine:3.19 | |
| volumes: | |
| - /runner-cache/apk-cache:/etc/apk/cache | |
| needs: [determine-version, build-and-push] | |
| if: ${{ needs.determine-version.outputs.should_release == 'true' }} | |
| permissions: | |
| contents: read | |
| packages: write | |
| steps: | |
| - name: Clean workspace | |
| run: | | |
| rm -rf "$GITHUB_WORKSPACE"/* || true | |
| rm -rf "$GITHUB_WORKSPACE"/.* || true | |
| - name: Install dependencies | |
| run: | | |
| apk add --no-cache git bash curl | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Configure Git for container | |
| run: | | |
| git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - name: Set up Helm | |
| uses: azure/setup-helm@v4 | |
| with: | |
| version: v3.14.0 | |
| - name: Log in to GHCR | |
| run: echo "${{ secrets.GITHUB_TOKEN }}" | helm registry login ghcr.io -u ${{ github.actor }} --password-stdin | |
| - name: Update Chart.yaml with release version | |
| run: | | |
| VERSION="${{ needs.determine-version.outputs.new_version }}" | |
| sed -i "s/^version: .*/version: ${VERSION}/" deploy/helm/Chart.yaml | |
| sed -i "s/^appVersion: .*/appVersion: \"v${VERSION}\"/" deploy/helm/Chart.yaml | |
| cat deploy/helm/Chart.yaml | |
| - name: Package Helm chart | |
| run: helm package deploy/helm --destination . | |
| - name: Push Helm chart to GHCR | |
| run: | | |
| VERSION="${{ needs.determine-version.outputs.new_version }}" | |
| helm push databasus-${VERSION}.tgz oci://ghcr.io/databasus/charts |