ci: visual tests (#2534)

* ci: testing visual comp * ci: more test * test: changes * chore: update CI * chore: fix npm * save * chore: working locally * chore: working logic * chore: save * chore: testing CI * ci: test ci * ci: more test * ci: more ci tests * ci: test * chore: updated test * chore: test * more tests * ci: test * ci: change folder structure * chore: fixed ci * chore: reduce test feedback * chore: more test * ci: path change * ci: more test * ci: update tar config * ci: retrest * chore: test * chore: test * ci: more test * chore: more test * save * ci: different workflow * chore: fixed logic * chore: more tests * core: ready for PR test * ci: added missing steps * ci: screenshot logic * ci: trigger * ci: more logic * ci: fix test path * test: new logic * ci: fix download path * chore: ready * save * ci: more tests * ci: fix screenshots * ci: more changes * chore: more tests * ci: more tests * chore: restart * chore: update * ci: removed Appzi from builds * chore: save * ci: new test * save * ci: updated artifacts logic * chore: verify self-hosted runner * chore: ready for merge (cherry picked from commit 0670689)
spectrocloud · Apr 2, 2024 · 0fecc23 · 0fecc23
1 parent be1627a
commit 0fecc23
Show file tree

Hide file tree

Showing 17 changed files with 1,165 additions and 89 deletions.
diff --git a/.dockerignore b/.dockerignore
@@ -4,4 +4,8 @@ CHANGES
 .gitignore
 .cache
 prow/
-.env
+.env
+screenshots/
+tests/screenshot.spec.ts-snapshots/
+tests-results/
+playwright-report/
diff --git a/.github/workflows/clean-up-report.yaml b/.github/workflows/clean-up-report.yaml
@@ -0,0 +1,61 @@
+name: Delete Visual Tests Reports
+
+on:
+  delete:
+    branches-ignore: [master, main, gh-pages]
+
+
+concurrency:
+  group: ${{ github.event.ref }}
+  cancel-in-progress: true
+
+jobs:
+  delete_reports:
+    name: Delete Reports
+    runs-on: ubuntu-latest
+    env:
+      # Contains all reports for deleted branch
+      BRANCH_REPORTS_DIR: reports/${{ github.event.ref }}
+    steps:
+      - name: Checkout GitHub Pages Branch
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+
+      - name: Set Git User
+        # see: https://github.com/actions/checkout/issues/13#issuecomment-724415212
+        run: |
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
+
+      - name: Check for workflow reports
+        run: |
+          if [ -z "$(ls -A $BRANCH_REPORTS_DIR)" ]; then
+            echo "BRANCH_REPORTS_EXIST="false"" >> $GITHUB_ENV
+          else
+            echo "BRANCH_REPORTS_EXIST="true"" >> $GITHUB_ENV
+          fi
+          
+      - name: Delete reports from repo for branch
+        if: ${{ env.BRANCH_REPORTS_EXIST == 'true' }}
+        timeout-minutes: 3
+        run: |
+          cd $BRANCH_REPORTS_DIR/..
+
+          rm -rf ${{ github.event.ref }}
+          git add .
+          git commit -m "workflow: remove all reports for branch ${{ github.event.ref }}"
+
+          while true; do
+            git pull --rebase
+            if [ $? -ne 0 ]; then
+              echo "Failed to rebase. Please review manually."
+              exit 1
+            fi
+
+            git push
+            if [ $? -eq 0 ]; then
+              echo "Successfully pushed HTML reports to repo."
+              exit 0
+            fi
+          done
diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml
@@ -0,0 +1,132 @@
+name: Screenshot Capture
+# This workflow is triggered when a workflow run of the "Release to Production" workflow is completed or when manually triggered.
+# The primary purpose of this workflow is to take screenshots of the website using Playwright and upload them as artifacts.
+# The screenshots can be used to compare the visual appearance of the website before and after a release.
+# The workflow Visual Comparison uses these screenshots to generate a visual comparison report.
+
+on:
+  workflow_run:
+    workflows: ["Release to Production"]
+    types: [completed]
+  workflow_dispatch:
+
+
+
+env:
+  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+  MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }}
+  FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }}
+  ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }}
+  ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }}
+  ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }}
+
+
+jobs:
+
+  create-assets:
+    name: asset-builds
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js Environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+          cache: "npm"
+
+      - name: Install Dependencies
+        run: npm ci
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium
+
+      - name: Build Website
+        run: make build
+
+      - name: Upload Build
+        uses: actions/upload-artifact@v4
+        with:
+          name: "build"
+          path: |
+            build/
+          if-no-files-found: error
+          retention-days: 1
+
+
+
+
+  visual-snapshots:
+    runs-on: ubuntu-latest
+    needs: [create-assets]
+    strategy:
+      fail-fast: false
+      matrix:
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js Environment
+        uses: actions/setup-node@v4
+        with:
+          node-version: "18"
+          cache: "npm"
+
+      - name: Install Dependencies
+        uses: Wandalen/wretry.action@v3
+        with: 
+            command: npm ci
+            attempt_limit: 3
+            attempt_delay: 60000 # 1 minute
+
+      - name: Install Playwright browsers
+        run: npx playwright install --with-deps chromium
+
+      - name: retry action
+        uses: Wandalen/wretry.action@v3
+        with: 
+          attempt_limit: 3
+          action: actions/download-artifact@v4
+          with: |
+            name: build
+            path: build
+          attempt_delay: 60000 # 1 minute
+
+      - name: Take Screenshots with Playwright
+        run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0
+
+      - name: Upload Screenshots
+        uses: actions/upload-artifact@v4
+        id: screenshots
+        with:
+          name: screenshots-${{ matrix.shardIndex }}
+          path: |
+            screenshots/
+          if-no-files-found: error
+          retention-days: 1
+
+  merge-snapshots:
+    name: Merge Screenshots
+    runs-on: ubuntu-latest
+    needs: [visual-snapshots]
+    steps:
+
+    - name: Download blob reports from GitHub Actions Artifacts
+      uses: actions/download-artifact@v4
+      with:
+        path: screenshots
+        pattern: screenshots-*
+        merge-multiple: true
+
+    - name: Upload Screenshots
+      uses: actions/upload-artifact@v4
+      id: screenshots
+      with:
+        name: "screenshots"
+        path: |
+          screenshots/
+        if-no-files-found: error
+        retention-days: 3