From b7e0248ba8527d04438265d74aeb40be180ba2ed Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 08:41:32 -0700 Subject: [PATCH 01/58] ci: testing visual comp --- .github/workflows/visual-comparison.yaml | 32 +++++++++++++ package-lock.json | 60 ++++++++++++++++++++++++ package.json | 5 +- tests/playwright.config.ts | 23 +++++++++ tests/screenshot.css | 35 ++++++++++++++ tests/screenshot.spec.ts | 37 +++++++++++++++ 6 files changed, 190 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/visual-comparison.yaml create mode 100644 tests/playwright.config.ts create mode 100644 tests/screenshot.css create mode 100644 tests/screenshot.spec.ts diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml new file mode 100644 index 0000000000..49d2828d5d --- /dev/null +++ b/.github/workflows/visual-comparison.yaml @@ -0,0 +1,32 @@ +name: Argos CI Screenshots + +on: + push: + branches: [visual-tests] + +jobs: + take-screenshots: + runs-on: ubuntu-latest + steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Use Node.js + uses: actions/setup-node@v3 + with: + node-version: current + + - name: Install dependencies + run: npm ci + + - name: Install Playwright browsers + run: yarn playwright install --with-deps chromium + + - name: Build the website + run: yarn docusaurus build + + - name: Take screenshots with Playwright + run: yarn playwright test + + # - name: Upload screenshots to Argos + # run: yarn argos upload ./screenshots \ No newline at end of file diff --git a/package-lock.json b/package-lock.json index f3d60d848b..327aa9f4b7 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,7 @@ "sass": "^1.72.0" }, "devDependencies": { + "@playwright/test": "^1.42.1", "@semantic-release/changelog": "^6.0.1", "@semantic-release/exec": "^6.0.3", "@semantic-release/git": "latest", @@ -5035,6 +5036,21 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/@playwright/test": { + "version": "1.42.1", + "resolved": "https://registry.npmjs.org/@playwright/test/-/test-1.42.1.tgz", + "integrity": "sha512-Gq9rmS54mjBL/7/MvBaNOBwbfnh7beHvS6oS4srqXFcQHpQCV1+c8JXWE8VLPyRDhgS3H8x8A7hztqI9VnwrAQ==", + "dev": true, + "dependencies": { + "playwright": "1.42.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, "node_modules/@pnpm/config.env-replace": { "version": "1.1.0", "license": "MIT", @@ -49728,6 +49744,50 @@ "node": ">=4" } }, + "node_modules/playwright": { + "version": "1.42.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.42.1.tgz", + "integrity": "sha512-PgwB03s2DZBcNRoW+1w9E+VkLBxweib6KTXM0M3tkiT4jVxKSi6PmVJ591J+0u10LUrgxB7dLRbiJqO5s2QPMg==", + "dev": true, + "dependencies": { + "playwright-core": "1.42.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=16" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.42.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.42.1.tgz", + "integrity": "sha512-mxz6zclokgrke9p1vtdy/COWBH+eOZgYUVVU34C73M+4j4HLlQJHtfcqiqqxpP0o8HhMkflvfbquLX5dg6wlfA==", + "dev": true, + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=16" + } + }, + "node_modules/playwright/node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "dev": true, + "hasInstallScript": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, "node_modules/plugin-error": { "version": "1.0.1", "dev": true, diff --git a/package.json b/package.json index 6132a5a531..71180cfcde 100644 --- a/package.json +++ b/package.json @@ -56,10 +56,11 @@ "prism-react-renderer": "^2.1.0", "react": "^18.2.0", "react-dom": "^18.2.0", - "sass": "^1.72.0", - "react-select": "^5.8.0" + "react-select": "^5.8.0", + "sass": "^1.72.0" }, "devDependencies": { + "@playwright/test": "^1.42.1", "@semantic-release/changelog": "^6.0.1", "@semantic-release/exec": "^6.0.3", "@semantic-release/git": "latest", diff --git a/tests/playwright.config.ts b/tests/playwright.config.ts new file mode 100644 index 0000000000..2d66a1d93a --- /dev/null +++ b/tests/playwright.config.ts @@ -0,0 +1,23 @@ +import { devices } from "@playwright/test"; +import type { PlaywrightTestConfig } from "@playwright/test"; + +/** + * See https://playwright.dev/docs/test-configuration. + */ +const config: PlaywrightTestConfig = { + webServer: { + cwd: "..", + port: 3000, + command: "yarn workspace website serve", + }, + projects: [ + { + name: "chromium", + use: { + ...devices["Desktop Chrome"], + }, + }, + ], +}; + +export default config; diff --git a/tests/screenshot.css b/tests/screenshot.css new file mode 100644 index 0000000000..0619db30be --- /dev/null +++ b/tests/screenshot.css @@ -0,0 +1,35 @@ +/* +We need to hide some elements in Argos/Playwright screenshots +Those elements are source of flakiness due to nondeterministic rendering +They don't consistently render exactly the same across CI runs + */ + +/******* DOCUSAURUS GLOBAL / THEME *******/ + +/* Iframes can load lazily */ +iframe, +/* Avatar images can be flaky due to using external sources: GitHub/Unavatar */ +.avatar__photo, +/* Gifs load lazily and are animated */ +img[src$='.gif'], +/* Algolia Keyboard shortcuts appear with a little delay */ +.DocSearch-Button-Keys > kbd, +/* The live playground preview can often display dates/counters */ +[class*='playgroundPreview'] { + visibility: hidden; +} + +/* +Different docs last-update dates can alter layout +"visibility: hidden" is not enough + */ +.theme-last-updated { + display: none; +} + +/* +Mermaid diagrams are rendered client-side and produce layout shifts + */ +.docusaurus-mermaid-container { + display: none; +} diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts new file mode 100644 index 0000000000..ee9045c13b --- /dev/null +++ b/tests/screenshot.spec.ts @@ -0,0 +1,37 @@ +import * as fs from "fs"; +import { test } from "@playwright/test"; +import { argosScreenshot } from "@argos-ci/playwright"; +import { extractSitemapPathnames, pathnameToArgosName } from "argos/utils"; + +// Constants: +const siteUrl = "http://localhost:3000"; +const sitemapPath = "../website/build/sitemap.xml"; +const stylesheetPath = "./screenshot.css"; +const stylesheet = fs.readFileSync(stylesheetPath).toString(); + +// Wait for hydration, requires Docusaurus v2.4.3+ +// See https://github.com/facebook/docusaurus/pull/9256 +// Docusaurus adds a once hydrated +function waitForDocusaurusHydration() { + return document.documentElement.dataset.hasHydrated === "true"; +} + +function isVersionedDocsPathname(pathname: string): boolean { + return pathname.match(/^\/docs\/((\d\.\d\.\d)|(next))\//); +} + +function screenshotPathname(pathname: string) { + test(`pathname ${pathname}`, async ({ page }) => { + const url = siteUrl + pathname; + await page.goto(url); + await page.waitForFunction(waitForDocusaurusHydration); + await page.addStyleTag({ content: stylesheet }); + await argosScreenshot(page, pathnameToArgosName(pathname)); + }); +} + +test.describe("Docusaurus site screenshots", () => { + const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); + console.log("Pathnames to screenshot:", pathnames); + pathnames.forEach(screenshotPathname); +}); From 8f8ca2e7dd3e9759ac0efe90e9eb096f40bd2be8 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 08:46:15 -0700 Subject: [PATCH 02/58] ci: more test --- .github/workflows/visual-comparison.yaml | 6 +++--- tests/screenshot.spec.ts | 22 ++++++++++------------ 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 49d2828d5d..b8c15f42cf 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -20,13 +20,13 @@ jobs: run: npm ci - name: Install Playwright browsers - run: yarn playwright install --with-deps chromium + run: npx playwright install --with-deps chromium - name: Build the website - run: yarn docusaurus build + run: make build - name: Take screenshots with Playwright - run: yarn playwright test + run: npx playwright test # - name: Upload screenshots to Argos # run: yarn argos upload ./screenshots \ No newline at end of file diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index ee9045c13b..173dc58026 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -1,7 +1,5 @@ import * as fs from "fs"; import { test } from "@playwright/test"; -import { argosScreenshot } from "@argos-ci/playwright"; -import { extractSitemapPathnames, pathnameToArgosName } from "argos/utils"; // Constants: const siteUrl = "http://localhost:3000"; @@ -20,18 +18,18 @@ function isVersionedDocsPathname(pathname: string): boolean { return pathname.match(/^\/docs\/((\d\.\d\.\d)|(next))\//); } -function screenshotPathname(pathname: string) { - test(`pathname ${pathname}`, async ({ page }) => { - const url = siteUrl + pathname; - await page.goto(url); - await page.waitForFunction(waitForDocusaurusHydration); - await page.addStyleTag({ content: stylesheet }); - await argosScreenshot(page, pathnameToArgosName(pathname)); - }); -} +// function screenshotPathname(pathname: string) { +// test(`pathname ${pathname}`, async ({ page }) => { +// const url = siteUrl + pathname; +// await page.goto(url); +// await page.waitForFunction(waitForDocusaurusHydration); +// await page.addStyleTag({ content: stylesheet }); +// await argosScreenshot(page, pathnameToArgosName(pathname)); +// }); +// } test.describe("Docusaurus site screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); console.log("Pathnames to screenshot:", pathnames); - pathnames.forEach(screenshotPathname); + // pathnames.forEach(screenshotPathname); }); From c1856c124922ff1c03167bb3c0b763a3d72a73ae Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 08:54:37 -0700 Subject: [PATCH 03/58] test: changes --- tests/screenshot.spec.ts | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index 173dc58026..ddc4929afe 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -1,6 +1,6 @@ import * as fs from "fs"; import { test } from "@playwright/test"; - +import { extractSitemapPathnames } from "argos/utils"; // Constants: const siteUrl = "http://localhost:3000"; const sitemapPath = "../website/build/sitemap.xml"; @@ -18,15 +18,16 @@ function isVersionedDocsPathname(pathname: string): boolean { return pathname.match(/^\/docs\/((\d\.\d\.\d)|(next))\//); } -// function screenshotPathname(pathname: string) { -// test(`pathname ${pathname}`, async ({ page }) => { -// const url = siteUrl + pathname; -// await page.goto(url); -// await page.waitForFunction(waitForDocusaurusHydration); -// await page.addStyleTag({ content: stylesheet }); -// await argosScreenshot(page, pathnameToArgosName(pathname)); -// }); -// } +function screenshotPathname(pathname: string) { + test(`pathname ${pathname}`, async ({ page }) => { + const url = siteUrl + pathname; + await page.goto(url); + await page.waitForFunction(waitForDocusaurusHydration); + await page.addStyleTag({ content: stylesheet }); + await expect(page).toHaveScreenshot(); + // await argosScreenshot(page, pathnameToArgosName(pathname)); + }); +} test.describe("Docusaurus site screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); From f8fbd163f4bad162141cdb19e575ec3732699e60 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 08:55:41 -0700 Subject: [PATCH 04/58] chore: update CI --- .github/workflows/visual-comparison.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index b8c15f42cf..2d8ebf6a25 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -4,6 +4,19 @@ on: push: branches: [visual-tests] + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }} + MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }} + FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }} + ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} + ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} + ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }} + jobs: take-screenshots: runs-on: ubuntu-latest From 013572bbaa49e298d1cae0c9c5c648724d4e0aa5 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 11:25:52 -0700 Subject: [PATCH 05/58] chore: fix npm --- .github/workflows/visual-comparison.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 2d8ebf6a25..631607497b 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -1,4 +1,4 @@ -name: Argos CI Screenshots +name: Visual Comparison on: push: @@ -24,10 +24,11 @@ jobs: - name: Check out repository code uses: actions/checkout@v4 - - name: Use Node.js - uses: actions/setup-node@v3 + - name: Setup Node.js environment + uses: actions/setup-node@v4 with: - node-version: current + node-version: "18" + cache: "npm" - name: Install dependencies run: npm ci From b24ac3abe01feddbed316619eed4da6c2f8f88b8 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 14:38:58 -0700 Subject: [PATCH 06/58] save --- package-lock.json | 393 ++++++++++++++++++++++++++++++--------- package.json | 2 + tests/screenshot.spec.ts | 11 +- tests/utils.ts | 17 ++ 4 files changed, 334 insertions(+), 89 deletions(-) create mode 100644 tests/utils.ts diff --git a/package-lock.json b/package-lock.json index 327aa9f4b7..70d3ce482b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -34,6 +34,8 @@ "sass": "^1.72.0" }, "devDependencies": { + "@argos-ci/cli": "^1.0.12", + "@argos-ci/playwright": "^2.0.0", "@playwright/test": "^1.42.1", "@semantic-release/changelog": "^6.0.1", "@semantic-release/exec": "^6.0.3", @@ -323,6 +325,108 @@ "url": "https://github.com/sponsors/philsturgeon" } }, + "node_modules/@argos-ci/browser": { + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/@argos-ci/browser/-/browser-1.5.0.tgz", + "integrity": "sha512-a9uzWL3L2qalqXa5kzkjlfnznpkGbzNhS5QSf0DtCylOWSrpnUMK+cIj3xzGMgmZK7SZ/r9OYdSZfsaSizWLXA==", + "dev": true, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@argos-ci/cli": { + "version": "1.0.12", + "resolved": "https://registry.npmjs.org/@argos-ci/cli/-/cli-1.0.12.tgz", + "integrity": "sha512-Hd/n6kuQwAbIMqfQg/+zzYzC4xoqekBCkfc695KdU2eFDnIokUcxyJmAnBDE9zD99VfaV2kw/0VGVAoAZnzEYA==", + "dev": true, + "dependencies": { + "@argos-ci/core": "1.5.5", + "commander": "^11.0.0", + "ora": "^7.0.1", + "update-notifier": "^6.0.2" + }, + "bin": { + "argos": "bin/argos-cli.js" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@argos-ci/cli/node_modules/commander": { + "version": "11.1.0", + "resolved": "https://registry.npmjs.org/commander/-/commander-11.1.0.tgz", + "integrity": "sha512-yPVavfyCcRhmorC7rWlkHn15b4wDVgVmBA7kV4QVBsF7kv/9TKJAbAXVTxvTnwP8HHKjRCJDClKbciiYS7p0DQ==", + "dev": true, + "engines": { + "node": ">=16" + } + }, + "node_modules/@argos-ci/core": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/@argos-ci/core/-/core-1.5.5.tgz", + "integrity": "sha512-m271rCtp5fnpUU89eSlS/If5WnDIUGduEHfW9YrwVHnMYWPCNBDxp2q87Wgjl3O5RTm/x+sDED3vFD9i1q+yTQ==", + "dev": true, + "dependencies": { + "@argos-ci/util": "1.2.1", + "axios": "^1.5.0", + "convict": "^6.2.4", + "debug": "^4.3.4", + "fast-glob": "^3.3.1", + "sharp": "^0.32.5", + "tmp": "^0.2.1" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@argos-ci/core/node_modules/axios": { + "version": "1.6.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.6.8.tgz", + "integrity": "sha512-v/ZHtJDU39mDpyBoFVkETcd/uNdxrWRrg3bKpOKzXFA6Bvqopts6ALSMU3y6ijYxbw2B+wPrIv46egTzJXCLGQ==", + "dev": true, + "dependencies": { + "follow-redirects": "^1.15.6", + "form-data": "^4.0.0", + "proxy-from-env": "^1.1.0" + } + }, + "node_modules/@argos-ci/playwright": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/@argos-ci/playwright/-/playwright-2.0.0.tgz", + "integrity": "sha512-rER3LPRYCS7CD3VK7ddP63tXj/3crQKZqrPGtY3QobyRO0IFech6xejEAHz2zM8ZT8aXH/dQydoU7vR9PCP8SQ==", + "dev": true, + "dependencies": { + "@argos-ci/browser": "1.5.0", + "@argos-ci/core": "1.5.5", + "@argos-ci/util": "1.2.1", + "chalk": "^5.3.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">=16.0.0" + } + }, + "node_modules/@argos-ci/playwright/node_modules/chalk": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", + "dev": true, + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/@argos-ci/util": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/@argos-ci/util/-/util-1.2.1.tgz", + "integrity": "sha512-/o7t0TcSED3BsBnnPrvXdmT+reThGMoGC9Lk+TTghrEE9GFlSKhjBmfYt4fUgXj5hQIe5tcdO01BVB2TsjjYSw==", + "dev": true, + "engines": { + "node": ">=16.0.0" + } + }, "node_modules/@babel/code-frame": { "version": "7.23.5", "license": "MIT", @@ -10291,6 +10395,18 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/cli-spinners": { + "version": "2.9.2", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.9.2.tgz", + "integrity": "sha512-ywqV+5MmyL4E7ybXgKys4DugZbX0FC6LnwrhjuykIjnK9k8OQacQ7axGKnjDXWNhns0xot3bZI5h55H8yo9cJg==", + "dev": true, + "engines": { + "node": ">=6" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/cli-table3": { "version": "0.6.3", "license": "MIT", @@ -10954,6 +11070,19 @@ "version": "2.0.0", "license": "MIT" }, + "node_modules/convict": { + "version": "6.2.4", + "resolved": "https://registry.npmjs.org/convict/-/convict-6.2.4.tgz", + "integrity": "sha512-qN60BAwdMVdofckX7AlohVJ2x9UvjTNoKVXCL2LxFk1l7757EJqf1nySdMkPQer0bt8kQ5lQiyZ9/2NvrFBuwQ==", + "dev": true, + "dependencies": { + "lodash.clonedeep": "^4.5.0", + "yargs-parser": "^20.2.7" + }, + "engines": { + "node": ">=6" + } + }, "node_modules/cookie": { "version": "0.5.0", "license": "MIT", @@ -21307,6 +21436,18 @@ "node": ">=10" } }, + "node_modules/is-interactive": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-2.0.0.tgz", + "integrity": "sha512-qP1vozQRI+BMOPcjFzrjXuQvdak2pHNUMZoeG2eRbiSqyvbEf/wQtEOTOX1guk6E3t36RkaqiSt8A/6YElNxLQ==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/is-jpg": { "version": "2.0.0", "dev": true, @@ -26234,6 +26375,34 @@ "version": "4.3.1", "license": "MIT" }, + "node_modules/log-symbols": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-5.1.0.tgz", + "integrity": "sha512-l0x2DvrW294C9uDCoQe1VSU4gf529FkSZ6leBl4TiqZH/e+0R7hSfHQBNut2mNygDgHwvYHfFLn6Oxb3VWj2rA==", + "dev": true, + "dependencies": { + "chalk": "^5.0.0", + "is-unicode-supported": "^1.1.0" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/log-symbols/node_modules/chalk": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", + "dev": true, + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, "node_modules/log-update": { "version": "5.0.1", "dev": true, @@ -35654,17 +35823,6 @@ "node": ">=4" } }, - "node_modules/netlify-cli/node_modules/cli-spinners": { - "version": "2.6.1", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=6" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/netlify-cli/node_modules/cli-truncate": { "version": "3.1.0", "dev": true, @@ -39581,17 +39739,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/netlify-cli/node_modules/is-interactive": { - "version": "2.0.0", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/netlify-cli/node_modules/is-npm": { "version": "6.0.0", "dev": true, @@ -40354,21 +40501,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/netlify-cli/node_modules/log-symbols": { - "version": "5.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "chalk": "^5.0.0", - "is-unicode-supported": "^1.1.0" - }, - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/netlify-cli/node_modules/log-update": { "version": "5.0.1", "dev": true, @@ -43604,53 +43736,6 @@ "dev": true, "license": "MIT" }, - "node_modules/netlify-cli/node_modules/stdin-discarder": { - "version": "0.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "bl": "^5.0.0" - }, - "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/netlify-cli/node_modules/stdin-discarder/node_modules/bl": { - "version": "5.1.0", - "dev": true, - "license": "MIT", - "dependencies": { - "buffer": "^6.0.3", - "inherits": "^2.0.4", - "readable-stream": "^3.4.0" - } - }, - "node_modules/netlify-cli/node_modules/stdin-discarder/node_modules/buffer": { - "version": "6.0.3", - "dev": true, - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "license": "MIT", - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "node_modules/netlify-cli/node_modules/streamx": { "version": "2.15.0", "dev": true, @@ -48875,6 +48960,91 @@ "node": ">=10" } }, + "node_modules/ora": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/ora/-/ora-7.0.1.tgz", + "integrity": "sha512-0TUxTiFJWv+JnjWm4o9yvuskpEJLXTcng8MJuKd+SzAzp2o+OP3HWqNhB4OdJRt1Vsd9/mR0oyaEYlOnL7XIRw==", + "dev": true, + "dependencies": { + "chalk": "^5.3.0", + "cli-cursor": "^4.0.0", + "cli-spinners": "^2.9.0", + "is-interactive": "^2.0.0", + "is-unicode-supported": "^1.3.0", + "log-symbols": "^5.1.0", + "stdin-discarder": "^0.1.0", + "string-width": "^6.1.0", + "strip-ansi": "^7.1.0" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora/node_modules/ansi-regex": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-6.0.1.tgz", + "integrity": "sha512-n5M855fKb2SsfMIiFFoVrABHJC8QtHwVx+mHWP3QcEqBHYienj5dHSgjbxtC0WEZXYt4wcD6zrQElDPhFuZgfA==", + "dev": true, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/ansi-regex?sponsor=1" + } + }, + "node_modules/ora/node_modules/chalk": { + "version": "5.3.0", + "resolved": "https://registry.npmjs.org/chalk/-/chalk-5.3.0.tgz", + "integrity": "sha512-dLitG79d+GV1Nb/VYcCDFivJeK1hiukt9QjRNVOsUtTy1rR1YJsmpGGTZ3qJos+uw7WmWF4wUwBd9jxjocFC2w==", + "dev": true, + "engines": { + "node": "^12.17.0 || ^14.13 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/chalk/chalk?sponsor=1" + } + }, + "node_modules/ora/node_modules/emoji-regex": { + "version": "10.3.0", + "resolved": "https://registry.npmjs.org/emoji-regex/-/emoji-regex-10.3.0.tgz", + "integrity": "sha512-QpLs9D9v9kArv4lfDEgg1X/gN5XLnf/A6l9cs8SPZLRZR3ZkY9+kwIQTxm+fsSej5UMYGE8fdoaZVIBlqG0XTw==", + "dev": true + }, + "node_modules/ora/node_modules/string-width": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/string-width/-/string-width-6.1.0.tgz", + "integrity": "sha512-k01swCJAgQmuADB0YIc+7TuatfNvTBVOoaUWJjTB9R4VJzR5vNWzf5t42ESVZFPS8xTySF7CAdV4t/aaIm3UnQ==", + "dev": true, + "dependencies": { + "eastasianwidth": "^0.2.0", + "emoji-regex": "^10.2.1", + "strip-ansi": "^7.0.1" + }, + "engines": { + "node": ">=16" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/ora/node_modules/strip-ansi": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz", + "integrity": "sha512-iq6eVVI64nQQTRYq2KtEg2d2uU7LElhTJwsH4YzIHZshxlgZms/wIc4VoDQTlG/IvVIrBKG06CrZnp0qv7hkcQ==", + "dev": true, + "dependencies": { + "ansi-regex": "^6.0.1" + }, + "engines": { + "node": ">=12" + }, + "funding": { + "url": "https://github.com/chalk/strip-ansi?sponsor=1" + } + }, "node_modules/ordered-read-streams": { "version": "1.0.1", "dev": true, @@ -50717,6 +50887,12 @@ "node": ">= 0.10" } }, + "node_modules/proxy-from-env": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", + "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==", + "dev": true + }, "node_modules/prr": { "version": "1.0.1", "dev": true, @@ -58903,6 +59079,46 @@ "version": "3.6.0", "license": "MIT" }, + "node_modules/stdin-discarder": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/stdin-discarder/-/stdin-discarder-0.1.0.tgz", + "integrity": "sha512-xhV7w8S+bUwlPTb4bAOUQhv8/cSS5offJuX8GQGq32ONF0ZtDWKfkdomM3HMRA+LhX6um/FZ0COqlwsjD53LeQ==", + "dev": true, + "dependencies": { + "bl": "^5.0.0" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/stdin-discarder/node_modules/bl": { + "version": "5.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-5.1.0.tgz", + "integrity": "sha512-tv1ZJHLfTDnXE6tMHv73YgSJaWR2AFuPwMntBe7XL/GBFHnT0CLnsHMogfk5+GzCDC5ZWarSCYaIGATZt9dNsQ==", + "dev": true, + "dependencies": { + "buffer": "^6.0.3", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/stdin-discarder/node_modules/readable-stream": { + "version": "3.6.2", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", + "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, "node_modules/stop-iteration-iterator": { "version": "1.0.0", "dev": true, @@ -60476,6 +60692,15 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/tmp": { + "version": "0.2.3", + "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.3.tgz", + "integrity": "sha512-nZD7m9iCPC5g0pYmcaxogYKggSfLsdxl8of3Q/oIbqCqLLIO9IAF0GWjX1z9NZRHPiXv8Wex4yDCaZsgEw0Y8w==", + "dev": true, + "engines": { + "node": ">=14.14" + } + }, "node_modules/tmpl": { "version": "1.0.5", "dev": true, diff --git a/package.json b/package.json index 71180cfcde..01a4c107a3 100644 --- a/package.json +++ b/package.json @@ -60,6 +60,8 @@ "sass": "^1.72.0" }, "devDependencies": { + "@argos-ci/cli": "^1.0.12", + "@argos-ci/playwright": "^2.0.0", "@playwright/test": "^1.42.1", "@semantic-release/changelog": "^6.0.1", "@semantic-release/exec": "^6.0.3", diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index ddc4929afe..74ca23c741 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -1,10 +1,10 @@ import * as fs from "fs"; import { test } from "@playwright/test"; -import { extractSitemapPathnames } from "argos/utils"; +import { extractSitemapPathnames } from "./utils"; // Constants: const siteUrl = "http://localhost:3000"; -const sitemapPath = "../website/build/sitemap.xml"; -const stylesheetPath = "./screenshot.css"; +const sitemapPath = "build/sitemap.xml"; +const stylesheetPath = "tests/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); // Wait for hydration, requires Docusaurus v2.4.3+ @@ -30,7 +30,8 @@ function screenshotPathname(pathname: string) { } test.describe("Docusaurus site screenshots", () => { - const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); + // const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); + const pathnames = extractSitemapPathnames(sitemapPath); console.log("Pathnames to screenshot:", pathnames); - // pathnames.forEach(screenshotPathname); + pathnames.forEach(screenshotPathname); }); diff --git a/tests/utils.ts b/tests/utils.ts new file mode 100644 index 0000000000..5e815f67b6 --- /dev/null +++ b/tests/utils.ts @@ -0,0 +1,17 @@ +import * as cheerio from "cheerio"; +import * as fs from "fs"; + +export function extractSitemapPathnames(sitemapPath: string): string[] { + const sitemap = fs.readFileSync(sitemapPath).toString(); + const $ = cheerio.load(sitemap, { xmlMode: true }); + const urls: string[] = []; + $("loc").each(function handleLoc() { + urls.push($(this).text()); + }); + return urls.map((url) => new URL(url).pathname); +} + +// Converts a pathname to a decent screenshot name +export function pathnameToArgosName(pathname: string): string { + return pathname.replace(/^\/|\/$/g, "") || "index"; +} From 7e73246c26766c9a0043a2bbb1f58c1153538ea9 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 16:02:22 -0700 Subject: [PATCH 07/58] chore: working locally --- .dockerignore | 6 ++++- .github/workflows/visual-comparison.yaml | 5 +++- .gitignore | 8 ++++++- Makefile | 3 +++ playwright.config.ts | 18 ++++++++++++++ tests/screenshot.css | 8 +++++++ tests/screenshot.spec.ts | 30 +++++++++++++++++++----- 7 files changed, 69 insertions(+), 9 deletions(-) create mode 100644 playwright.config.ts diff --git a/.dockerignore b/.dockerignore index c75f282c3f..a2e0abb3d8 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,4 +4,8 @@ CHANGES .gitignore .cache prow/ -.env \ No newline at end of file +.env +screenshots/ +tests/screenshot.spec.ts-snapshots/ +tests-results/ +playwright-report/ diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 631607497b..5036a46730 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -3,7 +3,10 @@ name: Visual Comparison on: push: branches: [visual-tests] - +# on: +# push: +# paths: +# - 'src/**' env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore index 42fd0e41ee..87f5efb87f 100644 --- a/.gitignore +++ b/.gitignore @@ -52,4 +52,10 @@ staging_sidebars/ ## Cosign -cosign.key \ No newline at end of file +cosign.key + +# Visual Compare +screenshots/ +tests/screenshot.spec.ts-snapshots/ +tests-results/ +playwright-report/ \ No newline at end of file diff --git a/Makefile b/Makefile index bb532f940c..adc45c8dd7 100644 --- a/Makefile +++ b/Makefile @@ -65,6 +65,9 @@ api: ## Generate API docs test: ## Run Jest tests npm test +test-visual: ## Run visual regression tests + npx playwright test tests/screenshots.spec.ts --workers 4 + ##@ Git Targets commit: ## Add a Git commit. Usage: make commit MESSAGE="" diff --git a/playwright.config.ts b/playwright.config.ts new file mode 100644 index 0000000000..8694812a29 --- /dev/null +++ b/playwright.config.ts @@ -0,0 +1,18 @@ +// playwright.config.ts +import { PlaywrightTestConfig } from "@playwright/test"; + +const config: PlaywrightTestConfig = { + // Other config options... + snapshotDir: "screenshots/", + testDir: "tests", + fullyParallel: true, + workers: process.env.CI ? 1 : 2, + reporter: [["html", { open: "never" }]], + webServer: { + command: "npm run serve", + url: "http://127.0.0.1:3000", + reuseExistingServer: !process.env.CI, + }, +}; + +export default config; diff --git a/tests/screenshot.css b/tests/screenshot.css index 0619db30be..d43b23f97a 100644 --- a/tests/screenshot.css +++ b/tests/screenshot.css @@ -33,3 +33,11 @@ Mermaid diagrams are rendered client-side and produce layout shifts .docusaurus-mermaid-container { display: none; } + +/* +Hide Cookie Consent banner + +*/ +#uc-center-container { + display: none !important; +} diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index 74ca23c741..224b7aa5b3 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -1,5 +1,5 @@ import * as fs from "fs"; -import { test } from "@playwright/test"; +import { test, expect } from "@playwright/test"; import { extractSitemapPathnames } from "./utils"; // Constants: const siteUrl = "http://localhost:3000"; @@ -15,7 +15,17 @@ function waitForDocusaurusHydration() { } function isVersionedDocsPathname(pathname: string): boolean { - return pathname.match(/^\/docs\/((\d\.\d\.\d)|(next))\//); + // First, check if it starts with /api/, if so, return false immediately + if (pathname.startsWith("/api/")) { + return false; + } + // This regex checks if the pathname does not follow the /4.1.x/ pattern + // and still follows the allowed versioning pattern. + const isValidVersion = !pathname.match(/^\/docs\/(\d+\.\d+\.x)\//); + const isNextVersion = pathname.match(/^\/docs\/next\//); + + // Return true if it's a valid version format or the next version, otherwise false + return isValidVersion || isNextVersion ? true : false; } function screenshotPathname(pathname: string) { @@ -24,14 +34,22 @@ function screenshotPathname(pathname: string) { await page.goto(url); await page.waitForFunction(waitForDocusaurusHydration); await page.addStyleTag({ content: stylesheet }); - await expect(page).toHaveScreenshot(); - // await argosScreenshot(page, pathnameToArgosName(pathname)); + + // Sanitize the pathname to be used as a valid filename + // const sanitizedPathname = sanitizePathnameForFile(pathname); + + // Ensure the screenshot is saved with a cleaned-up, valid file name + // await page.screenshot({ path: `screenshots/${sanitizedPathname}.png`, fullPage: true }); + + // This line ensures that the screenshot matches the expected screenshot. + // It may need to be customized based on how your testing framework expects to + // handle screenshot comparisons. + await expect(page).toHaveScreenshot({ fullPage: true, timeout: 10000 }); }); } test.describe("Docusaurus site screenshots", () => { - // const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); - const pathnames = extractSitemapPathnames(sitemapPath); + const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); console.log("Pathnames to screenshot:", pathnames); pathnames.forEach(screenshotPathname); }); From 990d60792667387346be14e69b2ea56c876ef91e Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 16:33:26 -0700 Subject: [PATCH 08/58] chore: working logic --- .gitignore | 2 +- Makefile | 2 +- tests/screenshot.css | 4 ++-- tests/screenshot.spec.ts | 1 + 4 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 87f5efb87f..d1d71dcb29 100644 --- a/.gitignore +++ b/.gitignore @@ -57,5 +57,5 @@ cosign.key # Visual Compare screenshots/ tests/screenshot.spec.ts-snapshots/ -tests-results/ +test-results/ playwright-report/ \ No newline at end of file diff --git a/Makefile b/Makefile index adc45c8dd7..be6e164a4f 100644 --- a/Makefile +++ b/Makefile @@ -66,7 +66,7 @@ test: ## Run Jest tests npm test test-visual: ## Run visual regression tests - npx playwright test tests/screenshots.spec.ts --workers 4 + npx playwright test tests/screenshot.spec.ts --workers 2 ##@ Git Targets diff --git a/tests/screenshot.css b/tests/screenshot.css index d43b23f97a..292a70aaf8 100644 --- a/tests/screenshot.css +++ b/tests/screenshot.css @@ -1,5 +1,5 @@ /* -We need to hide some elements in Argos/Playwright screenshots +We need to hide some elements in Playwright screenshots Those elements are source of flakiness due to nondeterministic rendering They don't consistently render exactly the same across CI runs */ @@ -38,6 +38,6 @@ Mermaid diagrams are rendered client-side and produce layout shifts Hide Cookie Consent banner */ -#uc-center-container { +#usercentrics-root { display: none !important; } diff --git a/tests/screenshot.spec.ts b/tests/screenshot.spec.ts index 224b7aa5b3..7b7f0080fc 100644 --- a/tests/screenshot.spec.ts +++ b/tests/screenshot.spec.ts @@ -34,6 +34,7 @@ function screenshotPathname(pathname: string) { await page.goto(url); await page.waitForFunction(waitForDocusaurusHydration); await page.addStyleTag({ content: stylesheet }); + await page.waitForTimeout(1000); // Waits for 100 milliseconds // Sanitize the pathname to be used as a valid filename // const sanitizedPathname = sanitizePathnameForFile(pathname); From 376ef3974a633e22155135d57301c88b3586f6b7 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 18:47:01 -0700 Subject: [PATCH 09/58] chore: save --- .github/workflows/visual-comparison.yaml | 23 ++++++++++++++++++- Makefile | 12 +++++++++- .../devx/services/service-listings/vault.md | 2 +- playwright.config.ts | 10 +++++++- static/assets/icons/lock.svg | 12 ++++++++++ tests/playwright.config.ts | 23 ------------------- {tests => visuals}/screenshot.css | 0 {tests => visuals}/screenshot.spec.ts | 5 ++-- {tests => visuals}/utils.ts | 0 9 files changed, 58 insertions(+), 29 deletions(-) create mode 100644 static/assets/icons/lock.svg delete mode 100644 tests/playwright.config.ts rename {tests => visuals}/screenshot.css (100%) rename {tests => visuals}/screenshot.spec.ts (93%) rename {tests => visuals}/utils.ts (100%) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 5036a46730..e1f8a95c26 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -43,7 +43,28 @@ jobs: run: make build - name: Take screenshots with Playwright - run: npx playwright test + run: make test-visual + + - name: Upload screenshots + uses: actions/upload-artifact@v4 + with: + name: "snapshots" + path: | + screenshots/screenshot.specs.ts-snapshots/ + if-no-files-found: error + retention-days: 1 + + - name: Upload Report + uses: actions/upload-artifact@v4 + with: + name: "report" + path: | + playwright-report/ + if-no-files-found: error + retention-days: 1 + + + # - name: Upload screenshots to Argos # run: yarn argos upload ./screenshots \ No newline at end of file diff --git a/Makefile b/Makefile index be6e164a4f..5f8bd228a4 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,10 @@ CHANGED_FILE=$(shell git diff-tree -r --no-commit-id --name-only master HEAD | g TEMP_DIR=$(shell $TMPDIR) +CPUS := $(shell sysctl -n hw.ncpu | awk '{print int($$1 / 2)}') + + + help: ## Display this help @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[0m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) @@ -31,6 +35,12 @@ clean-versions: ## Clean Docusarus content versions rm -rf api_versions.json versions.json versioned_docs versioned_sidebars api_versioned_sidebars api_versioned_docs git checkout -- docusaurus.config.js static/robots.txt +clean-visuals: + @echo "Cleaning visual regression tests" + + rm -rf test-results/ playwright-report/ screenshots/ + + ##@ npm Targets init: ## Initialize npm dependencies @@ -66,7 +76,7 @@ test: ## Run Jest tests npm test test-visual: ## Run visual regression tests - npx playwright test tests/screenshot.spec.ts --workers 2 + npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) ##@ Git Targets diff --git a/docs/docs-content/devx/services/service-listings/vault.md b/docs/docs-content/devx/services/service-listings/vault.md index 50180b7625..e7427c4c32 100644 --- a/docs/docs-content/devx/services/service-listings/vault.md +++ b/docs/docs-content/devx/services/service-listings/vault.md @@ -7,7 +7,7 @@ type: "appTier" category: ["security"] hiddenFromNav: false sidebar_position: 40 -logoUrl: "https://icon-library.com/images/padlock-icon.webp/padlock-icon.webp-29.jpg" +logoUrl: "/assets/icons/lock.svg" --- ## Vault diff --git a/playwright.config.ts b/playwright.config.ts index 8694812a29..6d26c3b9d2 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -4,8 +4,16 @@ import { PlaywrightTestConfig } from "@playwright/test"; const config: PlaywrightTestConfig = { // Other config options... snapshotDir: "screenshots/", - testDir: "tests", + testDir: "visuals", fullyParallel: true, + expect: { + toMatchSnapshot: { + maxDiffPixels: 10, + }, + toHaveScreenshot: { + maxDiffPixels: 10, + }, + }, workers: process.env.CI ? 1 : 2, reporter: [["html", { open: "never" }]], webServer: { diff --git a/static/assets/icons/lock.svg b/static/assets/icons/lock.svg new file mode 100644 index 0000000000..a76443f4ee --- /dev/null +++ b/static/assets/icons/lock.svg @@ -0,0 +1,12 @@ + + + + + + + \ No newline at end of file diff --git a/tests/playwright.config.ts b/tests/playwright.config.ts deleted file mode 100644 index 2d66a1d93a..0000000000 --- a/tests/playwright.config.ts +++ /dev/null @@ -1,23 +0,0 @@ -import { devices } from "@playwright/test"; -import type { PlaywrightTestConfig } from "@playwright/test"; - -/** - * See https://playwright.dev/docs/test-configuration. - */ -const config: PlaywrightTestConfig = { - webServer: { - cwd: "..", - port: 3000, - command: "yarn workspace website serve", - }, - projects: [ - { - name: "chromium", - use: { - ...devices["Desktop Chrome"], - }, - }, - ], -}; - -export default config; diff --git a/tests/screenshot.css b/visuals/screenshot.css similarity index 100% rename from tests/screenshot.css rename to visuals/screenshot.css diff --git a/tests/screenshot.spec.ts b/visuals/screenshot.spec.ts similarity index 93% rename from tests/screenshot.spec.ts rename to visuals/screenshot.spec.ts index 7b7f0080fc..984549e9f6 100644 --- a/tests/screenshot.spec.ts +++ b/visuals/screenshot.spec.ts @@ -4,7 +4,7 @@ import { extractSitemapPathnames } from "./utils"; // Constants: const siteUrl = "http://localhost:3000"; const sitemapPath = "build/sitemap.xml"; -const stylesheetPath = "tests/screenshot.css"; +const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); // Wait for hydration, requires Docusaurus v2.4.3+ @@ -33,8 +33,9 @@ function screenshotPathname(pathname: string) { const url = siteUrl + pathname; await page.goto(url); await page.waitForFunction(waitForDocusaurusHydration); + await page.waitForLoadState("domcontentloaded"); await page.addStyleTag({ content: stylesheet }); - await page.waitForTimeout(1000); // Waits for 100 milliseconds + await page.waitForTimeout(500); // Waits for 100 milliseconds // Sanitize the pathname to be used as a valid filename // const sanitizedPathname = sanitizePathnameForFile(pathname); diff --git a/tests/utils.ts b/visuals/utils.ts similarity index 100% rename from tests/utils.ts rename to visuals/utils.ts From 023bfd4b901430caa2e34ae483761df27f670a94 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 18:59:27 -0700 Subject: [PATCH 10/58] chore: testing CI --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index 5f8bd228a4..c00ffd4b75 100644 --- a/Makefile +++ b/Makefile @@ -78,6 +78,9 @@ test: ## Run Jest tests test-visual: ## Run visual regression tests npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) +test-visual-ci: ## Run visual regression tests + npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) + ##@ Git Targets commit: ## Add a Git commit. Usage: make commit MESSAGE="" From 6c6d2553337a653c7778c02608e2c3f4118a8b77 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 19:02:20 -0700 Subject: [PATCH 11/58] ci: test ci --- .github/workflows/visual-comparison.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index e1f8a95c26..0e0b66d496 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -22,7 +22,8 @@ env: jobs: take-screenshots: - runs-on: ubuntu-latest + name: Visual Comparison + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] steps: - name: Check out repository code uses: actions/checkout@v4 @@ -39,8 +40,8 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Build the website - run: make build + # - name: Build the website + # run: make build - name: Take screenshots with Playwright run: make test-visual From a7d46894d672ba2a0914573fe2322c4a5d6eeaa0 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 19:07:42 -0700 Subject: [PATCH 12/58] ci: more test --- .github/workflows/visual-comparison.yaml | 13 +++++++++---- Makefile | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 0e0b66d496..6d74f31079 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -40,14 +40,15 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - # - name: Build the website - # run: make build + - name: Build the website + run: make build - name: Take screenshots with Playwright run: make test-visual - name: Upload screenshots uses: actions/upload-artifact@v4 + id: screenshots with: name: "snapshots" path: | @@ -57,14 +58,18 @@ jobs: - name: Upload Report uses: actions/upload-artifact@v4 + id: report with: name: "report" path: | playwright-report/ if-no-files-found: error - retention-days: 1 - + retention-days: 1 + - name: URLs + run: | + echo 'Screenshots URL is ${{ steps.screenshots.outputs.artifact-url }}' + echo 'Report URL is ${{ steps.report.outputs.artifact-url }}' # - name: Upload screenshots to Argos diff --git a/Makefile b/Makefile index c00ffd4b75..a7213b156b 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test-visual: ## Run visual regression tests npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) test-visual-ci: ## Run visual regression tests - npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) + npx playwright test visuals/screenshot.spec.ts --workers 2 ##@ Git Targets From 23e7a769f8baf4039aa248d48d641bdacabe0eae Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 19:26:37 -0700 Subject: [PATCH 13/58] ci: more ci tests --- .github/workflows/visual-comparison.yaml | 8 ++------ Makefile | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 6d74f31079..8595b82d97 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -23,7 +23,7 @@ env: jobs: take-screenshots: name: Visual Comparison - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 @@ -44,7 +44,7 @@ jobs: run: make build - name: Take screenshots with Playwright - run: make test-visual + run: make test-visual-ci - name: Upload screenshots uses: actions/upload-artifact@v4 @@ -70,7 +70,3 @@ jobs: run: | echo 'Screenshots URL is ${{ steps.screenshots.outputs.artifact-url }}' echo 'Report URL is ${{ steps.report.outputs.artifact-url }}' - - - # - name: Upload screenshots to Argos - # run: yarn argos upload ./screenshots \ No newline at end of file diff --git a/Makefile b/Makefile index a7213b156b..ffaf50ce08 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test-visual: ## Run visual regression tests npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) test-visual-ci: ## Run visual regression tests - npx playwright test visuals/screenshot.spec.ts --workers 2 + npx playwright test visuals/screenshot.spec.ts --workers 4 ##@ Git Targets From 39ad322d2f737f2b8ba543af27880e8fb95b14a2 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 20:30:45 -0700 Subject: [PATCH 14/58] ci: test --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ffaf50ce08..303e2c0001 100644 --- a/Makefile +++ b/Makefile @@ -79,7 +79,7 @@ test-visual: ## Run visual regression tests npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) test-visual-ci: ## Run visual regression tests - npx playwright test visuals/screenshot.spec.ts --workers 4 + npx playwright test visuals/screenshot.spec.ts --workers 4 || exit 0 ##@ Git Targets From 72c48f24bc1c3a996cb6678368426e765cb99dc6 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Fri, 29 Mar 2024 21:09:09 -0700 Subject: [PATCH 15/58] chore: updated test --- .github/workflows/visual-comparison.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 8595b82d97..4889391463 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -52,7 +52,7 @@ jobs: with: name: "snapshots" path: | - screenshots/screenshot.specs.ts-snapshots/ + screenshots/screenshot.spec.ts-snapshots/ if-no-files-found: error retention-days: 1 From bd9111da8bd68ed91675ea2c814a98c409d130d9 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 06:43:05 -0700 Subject: [PATCH 16/58] chore: test --- .github/workflows/visual-comparison.yaml | 32 +++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 4889391463..c16ba2f472 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -20,6 +20,16 @@ env: ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }} + +concurrency: + group: "visual-comparison-${{ github.workflow }}" + cancel-in-progress: false + +permissions: + contents: read + pages: write + id-token: write + jobs: take-screenshots: name: Visual Comparison @@ -66,7 +76,27 @@ jobs: if-no-files-found: error retention-days: 1 - - name: URLs + - name: Download URLs run: | echo 'Screenshots URL is ${{ steps.screenshots.outputs.artifact-url }}' echo 'Report URL is ${{ steps.report.outputs.artifact-url }}' + + - name: Setup Pages + uses: actions/configure-pages@v5 + + - name: Unique Folder name + run : mkdir ${{ github.sha }} && cp -r playwright-report/* ${{ github.sha }} + + - name: Upload artifact + uses: actions/upload-pages-artifact@v3 + with: + path: ${{ github.sha }} + + - name: Deploy to GitHub Pages + id: deployment + uses: actions/deploy-pages@v4 + + - name: GH Pages URL + run: echo "https://spectrocloud.github.io/docs.spectrocloud.com/${{ github.sha }}/playwright-report/index.html" + + From 75347add62d2706fb79e1928f2ce4441aa2b4611 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 08:05:01 -0700 Subject: [PATCH 17/58] more tests --- .github/workflows/visual-comparison.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index c16ba2f472..7c32abe1d6 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -21,6 +21,7 @@ env: ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }} + concurrency: group: "visual-comparison-${{ github.workflow }}" cancel-in-progress: false @@ -32,6 +33,11 @@ permissions: jobs: take-screenshots: + + environment: + name: github-pages + url: ${{ steps.take-screenshots.outputs.page_url }} + name: Visual Comparison runs-on: ubuntu-latest steps: From 6be0f9805758d4450db0bd07dfd5f798bc74f7da Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 08:07:17 -0700 Subject: [PATCH 18/58] ci: test --- .github/workflows/visual-comparison.yaml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 7c32abe1d6..d07a025b71 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -26,10 +26,6 @@ concurrency: group: "visual-comparison-${{ github.workflow }}" cancel-in-progress: false -permissions: - contents: read - pages: write - id-token: write jobs: take-screenshots: @@ -38,6 +34,11 @@ jobs: name: github-pages url: ${{ steps.take-screenshots.outputs.page_url }} + permissions: + contents: read + pages: write + id-token: write + name: Visual Comparison runs-on: ubuntu-latest steps: From e133cdcd9e354e41a2fb26cfc673838d521b2b26 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 08:55:17 -0700 Subject: [PATCH 19/58] ci: change folder structure --- .github/workflows/visual-comparison.yaml | 17 +++++++++-------- visuals/screenshot.spec.ts | 23 ++++++++++++----------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index d07a025b71..026e2e4896 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -71,7 +71,7 @@ jobs: path: | screenshots/screenshot.spec.ts-snapshots/ if-no-files-found: error - retention-days: 1 + retention-days: 3 - name: Upload Report uses: actions/upload-artifact@v4 @@ -81,7 +81,7 @@ jobs: path: | playwright-report/ if-no-files-found: error - retention-days: 1 + retention-days: 3 - name: Download URLs run: | @@ -92,18 +92,19 @@ jobs: uses: actions/configure-pages@v5 - name: Unique Folder name - run : mkdir ${{ github.sha }} && cp -r playwright-report/* ${{ github.sha }} + run : mkdir -p ${{ github.sha }} && cp -r playwright-report/* ${{ github.sha }} - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: ${{ github.sha }} + + - name: Archive directory with SHA + run: tar -czvf visual-assets.tar ${GITHUB_SHA} - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 + with: + path: visual-assets.tar - name: GH Pages URL - run: echo "https://spectrocloud.github.io/docs.spectrocloud.com/${{ github.sha }}/playwright-report/index.html" + run: echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" diff --git a/visuals/screenshot.spec.ts b/visuals/screenshot.spec.ts index 984549e9f6..15daefa6e3 100644 --- a/visuals/screenshot.spec.ts +++ b/visuals/screenshot.spec.ts @@ -30,28 +30,29 @@ function isVersionedDocsPathname(pathname: string): boolean { function screenshotPathname(pathname: string) { test(`pathname ${pathname}`, async ({ page }) => { + console.log(`Taking screenshot of ${pathname}`); const url = siteUrl + pathname; await page.goto(url); await page.waitForFunction(waitForDocusaurusHydration); await page.waitForLoadState("domcontentloaded"); await page.addStyleTag({ content: stylesheet }); await page.waitForTimeout(500); // Waits for 100 milliseconds - - // Sanitize the pathname to be used as a valid filename - // const sanitizedPathname = sanitizePathnameForFile(pathname); - - // Ensure the screenshot is saved with a cleaned-up, valid file name - // await page.screenshot({ path: `screenshots/${sanitizedPathname}.png`, fullPage: true }); - - // This line ensures that the screenshot matches the expected screenshot. - // It may need to be customized based on how your testing framework expects to - // handle screenshot comparisons. await expect(page).toHaveScreenshot({ fullPage: true, timeout: 10000 }); }); } +// test.describe("cookie-banner is visible", () => { +// test("cookie-banner is visible", async ({ page }) => { +// await page.goto(siteUrl); +// await page.waitForFunction(waitForDocusaurusHydration); +// await page.waitForLoadState("domcontentloaded"); +// await page.addStyleTag({ content: stylesheet }); +// await expect(page).toHaveScreenshot({ fullPage: true }); +// await expect(page.getByTestId("#usercentrics-root")).toBeVisible(); +// }); +// }); + test.describe("Docusaurus site screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); - console.log("Pathnames to screenshot:", pathnames); pathnames.forEach(screenshotPathname); }); From a1d27f55c7dc722a575e9f8652094352a8c6855a Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 09:20:10 -0700 Subject: [PATCH 20/58] chore: fixed ci --- .github/workflows/visual-comparison.yaml | 2 +- Makefile | 4 +-- visuals/screenshot.api.spec.ts | 35 ++++++++++++++++++++ visuals/screenshot.spec.ts | 42 +++++++++--------------- visuals/utils.ts | 7 ++++ 5 files changed, 60 insertions(+), 30 deletions(-) create mode 100644 visuals/screenshot.api.spec.ts diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 026e2e4896..216932e3c9 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -102,7 +102,7 @@ jobs: id: deployment uses: actions/deploy-pages@v4 with: - path: visual-assets.tar + artifact_name: visual-assets.tar - name: GH Pages URL run: echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" diff --git a/Makefile b/Makefile index 303e2c0001..4f7ecbae58 100644 --- a/Makefile +++ b/Makefile @@ -76,10 +76,10 @@ test: ## Run Jest tests npm test test-visual: ## Run visual regression tests - npx playwright test visuals/screenshot.spec.ts --workers $(CPUS) + npx playwright test visuals/ --workers $(CPUS) test-visual-ci: ## Run visual regression tests - npx playwright test visuals/screenshot.spec.ts --workers 4 || exit 0 + npx playwright test visuals/ --workers 4 || exit 0 ##@ Git Targets diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts new file mode 100644 index 0000000000..0f8a8682c7 --- /dev/null +++ b/visuals/screenshot.api.spec.ts @@ -0,0 +1,35 @@ +import * as fs from "fs"; +import { test, expect } from "@playwright/test"; +import { extractSitemapPathnames, WaitForDocusaurusHydration } from "./utils"; +// Constants: +const siteUrl = "http://localhost:3000"; +const sitemapPath = "build/sitemap.xml"; +const stylesheetPath = "visuals/screenshot.css"; +const stylesheet = fs.readFileSync(stylesheetPath).toString(); + +function isApiDocsPathname(pathname: string): boolean { + // return false if the pathname does not start with /api/ + if (pathname.startsWith("/api/") && !pathname.match(/^\/api\/(\d+\.\d+\.x)\//)) { + return true; + } + return false; +} + +function screenshotPathname(pathname: string) { + test(`pathname ${pathname}`, async ({ page }) => { + console.log(`Taking screenshot of ${pathname}`); + const url = siteUrl + pathname; + await page.goto(url); + await page.waitForFunction(WaitForDocusaurusHydration); + await page.waitForLoadState("domcontentloaded"); + await page.addStyleTag({ content: stylesheet }); + await page.waitForTimeout(500); // Waits for 100 milliseconds + await expect(page).toHaveScreenshot({ fullPage: true, timeout: 10000 }); + }); +} + +test.describe("API docs screenshots", () => { + const pathnames = extractSitemapPathnames(sitemapPath).filter(isApiDocsPathname); + console.log("Total pathnames: ", pathnames.length); + pathnames.forEach(screenshotPathname); +}); diff --git a/visuals/screenshot.spec.ts b/visuals/screenshot.spec.ts index 15daefa6e3..c1db7c65e9 100644 --- a/visuals/screenshot.spec.ts +++ b/visuals/screenshot.spec.ts @@ -1,31 +1,18 @@ import * as fs from "fs"; import { test, expect } from "@playwright/test"; -import { extractSitemapPathnames } from "./utils"; +import { extractSitemapPathnames, WaitForDocusaurusHydration } from "./utils"; // Constants: const siteUrl = "http://localhost:3000"; const sitemapPath = "build/sitemap.xml"; const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); -// Wait for hydration, requires Docusaurus v2.4.3+ -// See https://github.com/facebook/docusaurus/pull/9256 -// Docusaurus adds a once hydrated -function waitForDocusaurusHydration() { - return document.documentElement.dataset.hasHydrated === "true"; -} - function isVersionedDocsPathname(pathname: string): boolean { - // First, check if it starts with /api/, if so, return false immediately - if (pathname.startsWith("/api/")) { + if (pathname.startsWith("/api/") || pathname.match(/\/\d+\.\d+\.x\//)) { return false; } - // This regex checks if the pathname does not follow the /4.1.x/ pattern - // and still follows the allowed versioning pattern. - const isValidVersion = !pathname.match(/^\/docs\/(\d+\.\d+\.x)\//); - const isNextVersion = pathname.match(/^\/docs\/next\//); - // Return true if it's a valid version format or the next version, otherwise false - return isValidVersion || isNextVersion ? true : false; + return true; } function screenshotPathname(pathname: string) { @@ -33,7 +20,7 @@ function screenshotPathname(pathname: string) { console.log(`Taking screenshot of ${pathname}`); const url = siteUrl + pathname; await page.goto(url); - await page.waitForFunction(waitForDocusaurusHydration); + await page.waitForFunction(WaitForDocusaurusHydration); await page.waitForLoadState("domcontentloaded"); await page.addStyleTag({ content: stylesheet }); await page.waitForTimeout(500); // Waits for 100 milliseconds @@ -41,18 +28,19 @@ function screenshotPathname(pathname: string) { }); } -// test.describe("cookie-banner is visible", () => { -// test("cookie-banner is visible", async ({ page }) => { -// await page.goto(siteUrl); -// await page.waitForFunction(waitForDocusaurusHydration); -// await page.waitForLoadState("domcontentloaded"); -// await page.addStyleTag({ content: stylesheet }); -// await expect(page).toHaveScreenshot({ fullPage: true }); -// await expect(page.getByTestId("#usercentrics-root")).toBeVisible(); -// }); -// }); +test.describe("cookie-banner is visible", () => { + test("cookie-banner is visible", async ({ page }) => { + await page.goto(siteUrl); + await page.waitForFunction(WaitForDocusaurusHydration); + await page.waitForLoadState("domcontentloaded"); + await page.addStyleTag({ content: stylesheet }); + await expect(page).toHaveScreenshot({ fullPage: true }); + await expect(page.getByTestId("#usercentrics-root")).toBeVisible(); + }); +}); test.describe("Docusaurus site screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); + console.log("Total pathnames: ", pathnames.length); pathnames.forEach(screenshotPathname); }); diff --git a/visuals/utils.ts b/visuals/utils.ts index 5e815f67b6..a524fc74e2 100644 --- a/visuals/utils.ts +++ b/visuals/utils.ts @@ -15,3 +15,10 @@ export function extractSitemapPathnames(sitemapPath: string): string[] { export function pathnameToArgosName(pathname: string): string { return pathname.replace(/^\/|\/$/g, "") || "index"; } + +// Wait for hydration, requires Docusaurus v2.4.3+ +// See https://github.com/facebook/docusaurus/pull/9256 +// Docusaurus adds a once hydrated +export function WaitForDocusaurusHydration() { + return document.documentElement.dataset.hasHydrated === "true"; +} From 979eb2349cc7d3e88227ec2f14100ea6d5f1efae Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 09:41:53 -0700 Subject: [PATCH 21/58] chore: reduce test feedback --- visuals/screenshot.api.spec.ts | 7 ++++- ...enshot.spec.ts => screenshot.docs.spec.ts} | 29 +++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) rename visuals/{screenshot.spec.ts => screenshot.docs.spec.ts} (61%) diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index 0f8a8682c7..1c344947ff 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -28,8 +28,13 @@ function screenshotPathname(pathname: string) { }); } +// test.describe("API docs screenshots", () => { +// const pathnames = extractSitemapPathnames(sitemapPath).filter(isApiDocsPathname); +// pathnames.forEach(screenshotPathname); +// }); + test.describe("API docs screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter(isApiDocsPathname); console.log("Total pathnames: ", pathnames.length); - pathnames.forEach(screenshotPathname); + screenshotPathname(pathnames[0]); }); diff --git a/visuals/screenshot.spec.ts b/visuals/screenshot.docs.spec.ts similarity index 61% rename from visuals/screenshot.spec.ts rename to visuals/screenshot.docs.spec.ts index c1db7c65e9..d6c023f554 100644 --- a/visuals/screenshot.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -28,19 +28,24 @@ function screenshotPathname(pathname: string) { }); } -test.describe("cookie-banner is visible", () => { - test("cookie-banner is visible", async ({ page }) => { - await page.goto(siteUrl); - await page.waitForFunction(WaitForDocusaurusHydration); - await page.waitForLoadState("domcontentloaded"); - await page.addStyleTag({ content: stylesheet }); - await expect(page).toHaveScreenshot({ fullPage: true }); - await expect(page.getByTestId("#usercentrics-root")).toBeVisible(); - }); -}); +// test.describe("cookie-banner is visible", () => { +// test("cookie-banner is visible", async ({ page }) => { +// await page.goto(siteUrl); +// await page.waitForFunction(WaitForDocusaurusHydration); +// await page.waitForLoadState("domcontentloaded"); +// await page.addStyleTag({ content: stylesheet }); +// await expect(page).toHaveScreenshot({ fullPage: true }); +// await expect(page.getByTestId("#usercentrics-root")).toBeVisible(); +// }); +// }); + +// test.describe("Docusaurus site screenshots", () => { +// const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); +// pathnames.forEach(screenshotPathname); +// }); -test.describe("Docusaurus site screenshots", () => { +test.describe("API docs screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); console.log("Total pathnames: ", pathnames.length); - pathnames.forEach(screenshotPathname); + screenshotPathname(pathnames[0]); }); From 91dff3b38073f339a5b4760d2312f7c22a63d149 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 10:24:40 -0700 Subject: [PATCH 22/58] chore: more test --- .github/workflows/visual-comparison.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 216932e3c9..b780ff61c9 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -69,7 +69,7 @@ jobs: with: name: "snapshots" path: | - screenshots/screenshot.spec.ts-snapshots/ + screenshots/ if-no-files-found: error retention-days: 3 From 9df7d23a9281fc6c563b7a433f2e7133c2feb809 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 10:52:56 -0700 Subject: [PATCH 23/58] ci: path change --- .github/workflows/visual-comparison.yaml | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index b780ff61c9..5d71d4e003 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -91,13 +91,22 @@ jobs: - name: Setup Pages uses: actions/configure-pages@v5 - - name: Unique Folder name + - name: Unique Folder Name run : mkdir -p ${{ github.sha }} && cp -r playwright-report/* ${{ github.sha }} - name: Archive directory with SHA run: tar -czvf visual-assets.tar ${GITHUB_SHA} + - name: Upload artifact with New Folder Structure + id: upload-artifact + uses: actions/upload-artifact@v4 + with: + name: visual-assets + path: visual-assets.tar + retention-days: 3 + if-no-files-found: error + - name: Deploy to GitHub Pages id: deployment uses: actions/deploy-pages@v4 @@ -105,6 +114,10 @@ jobs: artifact_name: visual-assets.tar - name: GH Pages URL - run: echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" + run: | + echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" + echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" + + From 622377e4c896f6dd10341fa2e65f29c1342fa39b Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 16:46:14 -0700 Subject: [PATCH 24/58] ci: more test --- .github/workflows/visual-comparison.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 5d71d4e003..15c8631d8b 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -111,13 +111,13 @@ jobs: id: deployment uses: actions/deploy-pages@v4 with: - artifact_name: visual-assets.tar + artifact_name: visual-assets - name: GH Pages URL run: | echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" - + From 918eee6ff2fbe5abfb7f4409d94a1c6a4f905b4b Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 17:00:42 -0700 Subject: [PATCH 25/58] ci: update tar config --- .github/workflows/visual-comparison.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 15c8631d8b..68ecbd1c63 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -96,7 +96,11 @@ jobs: - name: Archive directory with SHA - run: tar -czvf visual-assets.tar ${GITHUB_SHA} + run: | + tar --dereference --hard-dereference \ + --exclude=.git \ + --exclude=.github \ + -cvf visual-assets.tar ${GITHUB_SHA} - name: Upload artifact with New Folder Structure id: upload-artifact From 22b52adfc9c7c87ad2243afe5ea1e6f75ca98fed Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 17:37:24 -0700 Subject: [PATCH 26/58] ci: retrest --- .github/workflows/visual-comparison.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 68ecbd1c63..aff5ce321b 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -96,11 +96,12 @@ jobs: - name: Archive directory with SHA - run: | - tar --dereference --hard-dereference \ + run: | + tar -dereference --hard-dereference \ --exclude=.git \ --exclude=.github \ - -cvf visual-assets.tar ${GITHUB_SHA} + -cvf visual-assets.tar ${GITHUB_SHA} && \ + ls -ll - name: Upload artifact with New Folder Structure id: upload-artifact From a695f69bced9e0b181302a9eb06f34bb9611c9e5 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sat, 30 Mar 2024 17:50:06 -0700 Subject: [PATCH 27/58] chore: test --- .github/workflows/visual-comparison.yaml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index aff5ce321b..09230bb5ed 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -97,11 +97,10 @@ jobs: - name: Archive directory with SHA run: | - tar -dereference --hard-dereference \ + tar --dereference --hard-dereference \ --exclude=.git \ --exclude=.github \ - -cvf visual-assets.tar ${GITHUB_SHA} && \ - ls -ll + -cvf visual-assets.tar ${GITHUB_SHA} && ls -ll - name: Upload artifact with New Folder Structure id: upload-artifact From 4c7e4bd1188b03c778cbbb975981a6d40470e4c7 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 08:19:53 -0700 Subject: [PATCH 28/58] chore: test --- .github/workflows/visual-comparison.yaml | 49 ++++++++++++++---------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 09230bb5ed..ec64ada236 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -94,31 +94,38 @@ jobs: - name: Unique Folder Name run : mkdir -p ${{ github.sha }} && cp -r playwright-report/* ${{ github.sha }} - - - name: Archive directory with SHA - run: | - tar --dereference --hard-dereference \ - --exclude=.git \ - --exclude=.github \ - -cvf visual-assets.tar ${GITHUB_SHA} && ls -ll - - - name: Upload artifact with New Folder Structure - id: upload-artifact - uses: actions/upload-artifact@v4 - with: - name: visual-assets - path: visual-assets.tar - retention-days: 3 - if-no-files-found: error - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 + uses: peaceiris/actions-gh-pages@v3.7.3 with: - artifact_name: visual-assets + github_token: ${{ secrets.GITHUB_TOKEN }} + publish_dir: playwright-report/ + + + # - name: Archive directory with SHA + # run: | + # tar --dereference --hard-dereference \ + # --exclude=.git \ + # --exclude=.github \ + # -cvf visual-assets.tar ${GITHUB_SHA} && ls -ll + + # - name: Upload artifact with New Folder Structure + # id: upload-artifact + # uses: actions/upload-artifact@v4 + # with: + # name: visual-assets + # path: visual-assets.tar + # retention-days: 3 + # if-no-files-found: error + + # - name: Deploy to GitHub Pages + # id: deployment + # uses: actions/deploy-pages@v4 + # with: + # artifact_name: visual-assets + - name: GH Pages URL - run: | + run: | echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" From bffceee97f1e80c4f436c710a962ad10623b8f29 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 08:34:07 -0700 Subject: [PATCH 29/58] ci: more test --- .github/workflows/visual-comparison.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index ec64ada236..dddbe9cd46 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -35,7 +35,7 @@ jobs: url: ${{ steps.take-screenshots.outputs.page_url }} permissions: - contents: read + contents: write pages: write id-token: write @@ -99,7 +99,8 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: playwright-report/ - + + # - name: Archive directory with SHA # run: | From 6e0c103a1fd6c4a480690a3517d2edb84efe7740 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 08:48:29 -0700 Subject: [PATCH 30/58] chore: more test --- .github/workflows/visual-comparison.yaml | 37 +++++------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index dddbe9cd46..8d1b0a44e8 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -71,7 +71,7 @@ jobs: path: | screenshots/ if-no-files-found: error - retention-days: 3 + retention-days: 1 - name: Upload Report uses: actions/upload-artifact@v4 @@ -81,7 +81,7 @@ jobs: path: | playwright-report/ if-no-files-found: error - retention-days: 3 + retention-days: 1 - name: Download URLs run: | @@ -99,36 +99,15 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: playwright-report/ + destination_dir: ${{ github.sha }} + keep_files: true - - - # - name: Archive directory with SHA - # run: | - # tar --dereference --hard-dereference \ - # --exclude=.git \ - # --exclude=.github \ - # -cvf visual-assets.tar ${GITHUB_SHA} && ls -ll - - # - name: Upload artifact with New Folder Structure - # id: upload-artifact - # uses: actions/upload-artifact@v4 - # with: - # name: visual-assets - # path: visual-assets.tar - # retention-days: 3 - # if-no-files-found: error - - # - name: Deploy to GitHub Pages - # id: deployment - # uses: actions/deploy-pages@v4 - # with: - # artifact_name: visual-assets - - + - name: GH Pages URL run: | - echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" - echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" + # echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" + # echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" + echo "::notice title=📋 Published Playwright Test Report::https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" From 175f02fc5dbac4ad01040a5b068067562bf049ba Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 09:17:37 -0700 Subject: [PATCH 31/58] save --- .github/workflows/clean-up-report.yaml | 61 ++++++++++++++++++++++++ .github/workflows/visual-comparison.yaml | 43 ++++++++--------- 2 files changed, 82 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/clean-up-report.yaml diff --git a/.github/workflows/clean-up-report.yaml b/.github/workflows/clean-up-report.yaml new file mode 100644 index 0000000000..466ada9348 --- /dev/null +++ b/.github/workflows/clean-up-report.yaml @@ -0,0 +1,61 @@ +name: Delete + +on: + delete: + branches-ignore: [master, main, gh-pages] + + +concurrency: + group: ${{ github.event.ref }} + cancel-in-progress: true + +jobs: + delete_reports: + name: Delete Reports + runs-on: ubuntu-latest + env: + # Contains all reports for deleted branch + BRANCH_REPORTS_DIR: reports/${{ github.event.ref }} + steps: + - name: Checkout GitHub Pages Branch + uses: actions/checkout@v4 + with: + ref: gh-pages + + - name: Set Git User + # see: https://github.com/actions/checkout/issues/13#issuecomment-724415212 + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Check for workflow reports + run: | + if [ -z "$(ls -A $BRANCH_REPORTS_DIR)" ]; then + echo "BRANCH_REPORTS_EXIST="false"" >> $GITHUB_ENV + else + echo "BRANCH_REPORTS_EXIST="true"" >> $GITHUB_ENV + fi + + - name: Delete reports from repo for branch + if: ${{ env.BRANCH_REPORTS_EXIST == 'true' }} + timeout-minutes: 3 + run: | + cd $BRANCH_REPORTS_DIR/.. + + rm -rf ${{ github.event.ref }} + git add . + git commit -m "workflow: remove all reports for branch ${{ github.event.ref }}" + + while true; do + git pull --rebase + if [ $? -ne 0 ]; then + echo "Failed to rebase. Please review manually." + exit 1 + fi + + git push + if [ $? -eq 0 ]; then + echo "Successfully pushed HTML reports to repo." + exit 0 + fi + done \ No newline at end of file diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 8d1b0a44e8..7af734251d 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -33,6 +33,7 @@ jobs: environment: name: github-pages url: ${{ steps.take-screenshots.outputs.page_url }} + HTML_REPORT_URL_PATH: reports/${{ github.ref_name }}/${{ github.run_id }}/${{ github.run_attempt }} permissions: contents: write @@ -63,25 +64,25 @@ jobs: - name: Take screenshots with Playwright run: make test-visual-ci - - name: Upload screenshots - uses: actions/upload-artifact@v4 - id: screenshots - with: - name: "snapshots" - path: | - screenshots/ - if-no-files-found: error - retention-days: 1 - - - name: Upload Report - uses: actions/upload-artifact@v4 - id: report - with: - name: "report" - path: | - playwright-report/ - if-no-files-found: error - retention-days: 1 + - name: Push HTML Report + timeout-minutes: 3 + run: | + git add . + git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt: ${{ github.run_attempt }})" + while true; do + git pull --rebase + if [ $? -ne 0 ]; then + echo "Failed to rebase. Please review manually." + exit 1 + fi + + git push + if [ $? -eq 0 ]; then + echo "Successfully pushed HTML report to repo." + exit 0 + fi + done + - name: Download URLs run: | @@ -105,9 +106,7 @@ jobs: - name: GH Pages URL run: | - # echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/playwright-report/index.html" - # echo "https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" - echo "::notice title=📋 Published Playwright Test Report::https://spectrocloud.github.io/librarium/${{ github.sha }}/index.html" + echo "::notice title=📋 Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH" From a9bf13528b02e2d144380bcf7be49cc32b7b53a0 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 09:30:32 -0700 Subject: [PATCH 32/58] ci: different workflow --- .github/workflows/visual-comparison.yaml | 35 +----------------------- 1 file changed, 1 insertion(+), 34 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 7af734251d..a08646c3a8 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -19,6 +19,7 @@ env: ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }} + HTML_REPORT_URL_PATH: reports/${{ github.ref_name }}/${{ github.run_id }}/${{ github.run_attempt }} @@ -30,16 +31,6 @@ concurrency: jobs: take-screenshots: - environment: - name: github-pages - url: ${{ steps.take-screenshots.outputs.page_url }} - HTML_REPORT_URL_PATH: reports/${{ github.ref_name }}/${{ github.run_id }}/${{ github.run_attempt }} - - permissions: - contents: write - pages: write - id-token: write - name: Visual Comparison runs-on: ubuntu-latest steps: @@ -83,31 +74,7 @@ jobs: fi done - - - name: Download URLs - run: | - echo 'Screenshots URL is ${{ steps.screenshots.outputs.artifact-url }}' - echo 'Report URL is ${{ steps.report.outputs.artifact-url }}' - - - name: Setup Pages - uses: actions/configure-pages@v5 - - - name: Unique Folder Name - run : mkdir -p ${{ github.sha }} && cp -r playwright-report/* ${{ github.sha }} - - - name: Deploy to GitHub Pages - uses: peaceiris/actions-gh-pages@v3.7.3 - with: - github_token: ${{ secrets.GITHUB_TOKEN }} - publish_dir: playwright-report/ - destination_dir: ${{ github.sha }} - keep_files: true - - name: GH Pages URL run: | echo "::notice title=📋 Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH" - - - - From c22b634870e19ff4b354222334d3afd363e22a78 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 09:36:40 -0700 Subject: [PATCH 33/58] chore: fixed logic --- .github/workflows/visual-comparison.yaml | 92 ++++++++++++++++++------ 1 file changed, 69 insertions(+), 23 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index a08646c3a8..334275a46f 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -55,26 +55,72 @@ jobs: - name: Take screenshots with Playwright run: make test-visual-ci - - name: Push HTML Report - timeout-minutes: 3 - run: | - git add . - git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt: ${{ github.run_attempt }})" - while true; do - git pull --rebase - if [ $? -ne 0 ]; then - echo "Failed to rebase. Please review manually." - exit 1 - fi - - git push - if [ $? -eq 0 ]; then - echo "Successfully pushed HTML report to repo." - exit 0 - fi - done - - - - name: GH Pages URL - run: | - echo "::notice title=📋 Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH" + # - name: Upload screenshots + # uses: actions/upload-artifact@v4 + # id: screenshots + # with: + # name: "snapshots" + # path: | + # screenshots/ + # if-no-files-found: error + # retention-days: 1 + + - name: Upload Report + uses: actions/upload-artifact@v4 + id: report + with: + name: "report" + path: | + playwright-report/ + if-no-files-found: error + retention-days: 1 + + + publish_report: + name: Publish HTML Report + if: "success()" + needs: [test] + runs-on: ubuntu-latest + continue-on-error: true + steps: + + - name: Checkout GitHub Pages Branch + uses: actions/checkout@v4 + with: + ref: gh-pages + + - name: Set Git User + # see: https://github.com/actions/checkout/issues/13#issuecomment-724415212 + run: | + git config --global user.name "github-actions[bot]" + git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" + + - name: Download zipped HTML report + uses: actions/download-artifact@v4 + with: + name: report + path: ${{ env.HTML_REPORT_URL_PATH }} + + - name: Push HTML Report + timeout-minutes: 3 + run: | + git add . + git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt: ${{ github.run_attempt }})" + while true; do + git pull --rebase + if [ $? -ne 0 ]; then + echo "Failed to rebase. Please review manually." + exit 1 + fi + + git push + if [ $? -eq 0 ]; then + echo "Successfully pushed HTML report to repo." + exit 0 + fi + done + + + - name: GH Pages URL + run: | + echo "::notice title=📋 Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH" From 5021d0016825dd4aaea8fa5e57ccb97a0a9aa40f Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 09:47:35 -0700 Subject: [PATCH 34/58] chore: more tests --- .github/workflows/visual-comparison.yaml | 26 ++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 334275a46f..22499d6710 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -3,11 +3,16 @@ name: Visual Comparison on: push: branches: [visual-tests] + + # on: -# push: +# pull_request: +# types: [opened, reopened, synchronize, labeled] +# branches-ignore: ["version-*", "release-*"] # paths: # - 'src/**' + env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY }} @@ -29,9 +34,22 @@ concurrency: jobs: - take-screenshots: + # run-ci: + # # runs-on: ubuntu-latest + # runs-on: ubuntu-latest + # defaults: + # run: + # shell: bash + # if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' }} + # steps: + # # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped. + # # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed. + # # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI. + # - run: exit 0 + take-screenshots: name: Visual Comparison + # needs: [run-ci] runs-on: ubuntu-latest steps: - name: Check out repository code @@ -78,8 +96,8 @@ jobs: publish_report: name: Publish HTML Report - if: "success()" - needs: [test] + # needs: [run-ci, take-screenshots] + needs: [take-screenshots] runs-on: ubuntu-latest continue-on-error: true steps: From cf4cf20f4dd402f3b0cda7f05233c626db26eba0 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 10:15:05 -0700 Subject: [PATCH 35/58] core: ready for PR test --- .github/workflows/visual-comparison.yaml | 61 ++++++++++++++---------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 22499d6710..721139d255 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -1,16 +1,16 @@ name: Visual Comparison -on: - push: - branches: [visual-tests] +# on: +# push: +# branches: [visual-tests] -# on: -# pull_request: -# types: [opened, reopened, synchronize, labeled] -# branches-ignore: ["version-*", "release-*"] -# paths: -# - 'src/**' +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + branches-ignore: ["version-*", "release-*"] + paths: + - 'src/**' env: @@ -34,22 +34,22 @@ concurrency: jobs: - # run-ci: - # # runs-on: ubuntu-latest - # runs-on: ubuntu-latest - # defaults: - # run: - # shell: bash - # if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' }} - # steps: - # # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped. - # # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed. - # # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI. - # - run: exit 0 + run-ci: + # runs-on: ubuntu-latest + runs-on: ubuntu-latest + defaults: + run: + shell: bash + if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' || contains(github.event.pull_request.labels.*.name, 'visual-tests') }} + # steps: + # # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped. + # # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed. + # # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI. + # - run: exit 0 take-screenshots: name: Visual Comparison - # needs: [run-ci] + needs: [run-ci] runs-on: ubuntu-latest steps: - name: Check out repository code @@ -96,8 +96,7 @@ jobs: publish_report: name: Publish HTML Report - # needs: [run-ci, take-screenshots] - needs: [take-screenshots] + needs: [run-ci, take-screenshots] runs-on: ubuntu-latest continue-on-error: true steps: @@ -141,4 +140,16 @@ jobs: - name: GH Pages URL run: | - echo "::notice title=📋 Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH" + echo "::notice title= Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH" + + + - name: Post Netlify URL + uses: mshick/add-pr-comment@v2 + with: + message: | + 📋 Visual Report for ${{ github.ref_name }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready attempt at + https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH + message-failure: | + 👎 Uh oh! Unable to publish Visual Report URL. + refresh-message-position: true + update-only: true \ No newline at end of file From a294a9f91e03b326a39f333a03f3a8efe1da6898 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 10:15:56 -0700 Subject: [PATCH 36/58] ci: added missing steps --- .github/workflows/visual-comparison.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 721139d255..7ac3df17df 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -41,11 +41,11 @@ jobs: run: shell: bash if: ${{ !github.event.pull_request.draft && github.actor != 'dependabot[bot]' && github.actor != 'dependabot-preview[bot]' || contains(github.event.pull_request.labels.*.name, 'visual-tests') }} - # steps: - # # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped. - # # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed. - # # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI. - # - run: exit 0 + steps: + # If the condition above is not met, aka, the PR is not in draft status, then this step is skipped. + # Because this step is part of the critical path, omission of this step will result in remaining CI steps not gettinge executed. + # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI. + - run: exit 0 take-screenshots: name: Visual Comparison From d2fd0c1ac017c2ec082fed100e55954a8d640d50 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 13:59:24 -0700 Subject: [PATCH 37/58] ci: screenshot logic --- .github/workflows/screenshot_capture.yaml | 58 +++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100644 .github/workflows/screenshot_capture.yaml diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml new file mode 100644 index 0000000000..05d41724bf --- /dev/null +++ b/.github/workflows/screenshot_capture.yaml @@ -0,0 +1,58 @@ +name: Screenshot Capture +# This workflow is triggered when a workflow run of the "Release to Production" workflow is completed or when manually triggered. +# The primary purpose of this workflow is to take screenshots of the website using Playwright and upload them as artifacts. +# The screenshots can be used to compare the visual appearance of the website before and after a release. +# The workflow Visual Comparison uses these screenshots to generate a visual comparison report. + +on: + workflow_run: + workflows: ["Release to Production"] + types: [completed] + workflow_dispatch: + + +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }} + MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }} + FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }} + ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} + ALGOLIA_APP_ID: ${{ secrets.ALGOLIA_APP_ID }} + ALGOLIA_SEARCH_KEY: ${{ secrets.ALGOLIA_SEARCH_KEY }} + + +jobs: + + visual-snapshots: + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: "18" + cache: "npm" + + - name: Install dependencies + run: npm ci + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + + - name: Build the website + run: make build + + - name: Take screenshots with Playwright + run: make test-visual-ci + + - name: Upload screenshots + uses: actions/upload-artifact@v4 + id: screenshots + with: + name: "snapshots" + path: | + screenshots/ + if-no-files-found: error + retention-days: 3 \ No newline at end of file From 2026da20979a3cd738f985a95933e330d0ac470b Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 14:00:47 -0700 Subject: [PATCH 38/58] ci: trigger --- .github/workflows/screenshot_capture.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 05d41724bf..83431ccf7c 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -4,11 +4,15 @@ name: Screenshot Capture # The screenshots can be used to compare the visual appearance of the website before and after a release. # The workflow Visual Comparison uses these screenshots to generate a visual comparison report. +# on: +# workflow_run: +# workflows: ["Release to Production"] +# types: [completed] +# workflow_dispatch: + on: - workflow_run: - workflows: ["Release to Production"] - types: [completed] - workflow_dispatch: + push: + branches: [visual-tests] env: From 5b205e829956abef2220d0d96650ebf3e778efeb Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 15:02:07 -0700 Subject: [PATCH 39/58] ci: more logic --- .github/workflows/visual-comparison.yaml | 15 +-- scripts/screenshot_artifacts.sh | 114 +++++++++++++++++++++++ visuals/exclude.json | 1 + visuals/screenshot.api.spec.ts | 16 ++-- visuals/screenshot.docs.spec.ts | 19 ++-- 5 files changed, 136 insertions(+), 29 deletions(-) create mode 100755 scripts/screenshot_artifacts.sh create mode 100644 visuals/exclude.json diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 7ac3df17df..8f2f6b457a 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -70,19 +70,12 @@ jobs: - name: Build the website run: make build + - name: Download Reference Screenshots + run: ./scripts/screenshot_artifacts.sh + - name: Take screenshots with Playwright run: make test-visual-ci - # - name: Upload screenshots - # uses: actions/upload-artifact@v4 - # id: screenshots - # with: - # name: "snapshots" - # path: | - # screenshots/ - # if-no-files-found: error - # retention-days: 1 - - name: Upload Report uses: actions/upload-artifact@v4 id: report @@ -148,7 +141,7 @@ jobs: with: message: | 📋 Visual Report for ${{ github.ref_name }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready attempt at - https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH + https://spectrocloud.github.io/$GITHUB_REPOSITORY_OWNER/$HTML_REPORT_URL_PATH message-failure: | 👎 Uh oh! Unable to publish Visual Report URL. refresh-message-position: true diff --git a/scripts/screenshot_artifacts.sh b/scripts/screenshot_artifacts.sh new file mode 100755 index 0000000000..00c5b2b429 --- /dev/null +++ b/scripts/screenshot_artifacts.sh @@ -0,0 +1,114 @@ +#!/bin/bash + +#################################################################################################### +# This script is used to download the latest screenshots captured by the screenshot_capture workflow. +# The assumption is that this script is executed in a GitHub Actions environment. +# The script fetches the latest workflow run for the specified workflow file and repository. +# It then downloads the first artifact from the latest run and unzips it to a specified folder. +# The unzipped folder is named "screenshots" by default and dropped one level above the script location. +# The script requires the GITHUB_TOKEN environment variable to be set with a valid GitHub token. +# Otherwise the artifacts cannot be downloaded. +#################################################################################################### + + +set -e + +# Function to unzip the artifact +unzip_artifact() { + local zip_file=$1 + local destination_folder=$2 # Use passed variable for folder name + + echo "Unzipping $zip_file to $destination_folder..." + mkdir -p "$destination_folder" + unzip -o "$zip_file" -d "$destination_folder" + echo "Artifact unzipped successfully." + rm -f "$zip_file" +} + +# Ensure GITHUB_TOKEN is set +if [ -z "$GITHUB_TOKEN" ]; then + echo "GitHub token (GITHUB_TOKEN) not provided. Please set the GITHUB_TOKEN environment variable." + exit 1 +fi + + +OWNER="${GITHUB_REPOSITORY_OWNER:-spectrocloud}" +REPO_NAME="${GITHUB_REPOSITORY#*/}" +REPO="${REPO_NAME:-librarium}" +WORKFLOW_FILE="screenshot_capture.yaml" +UNZIP_FOLDER="screenshots" + + +perform_curl() { + local url=$1 + response=$(curl -s -f -w "%{http_code}" -o temp_response.txt -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3+json" "$url") + http_code=$(tail -n1 temp_response.txt) + if [ "$response" != "200" ]; then + echo "Error: HTTP response $http_code for $url" + cat temp_response.txt + rm temp_response.txt + exit 1 + fi + jq '.' < temp_response.txt + rm temp_response.txt +} + +# Fetch the latest workflow run for the specified workflow +LATEST_RUN=$(perform_curl "https://api.github.com/repos/$OWNER/$REPO/actions/workflows/$WORKFLOW_FILE/runs" | jq '.workflow_runs[0]') + +# Extract status, conclusion, and artifacts_url +STATUS=$(echo $LATEST_RUN | jq -r '.status') +CONCLUSION=$(echo $LATEST_RUN | jq -r '.conclusion') +ARTIFACTS_URL=$(echo $LATEST_RUN | jq -r '.artifacts_url') + +# Check if the status is "completed" and the conclusion is "success" +if [[ "$STATUS" == "completed" && "$CONCLUSION" == "success" ]]; then + echo "The latest workflow run is completed successfully." + + # Fetch artifacts details + ARTIFACTS=$(perform_curl "$ARTIFACTS_URL") + + # Parse and display artifact information + ARTIFACTS_COUNT=$(echo $ARTIFACTS | jq '.total_count') + echo "Found $ARTIFACTS_COUNT artifact(s)." + + if [[ "$ARTIFACTS_COUNT" -gt 0 ]]; then + # Loop through each artifact and print details + echo $ARTIFACTS | jq -c '.artifacts[]' | while read -r ARTIFACT; do + NAME=$(echo $ARTIFACT | jq -r '.name') + DOWNLOAD_URL=$(echo $ARTIFACT | jq -r '.archive_download_url') + SIZE=$(echo $ARTIFACT | jq -r '.size_in_bytes') + CREATED_AT=$(echo $ARTIFACT | jq -r '.created_at') + EXPIRES_AT=$(echo $ARTIFACT | jq -r '.expires_at') + + echo "Artifact: $NAME" + echo "Size: $SIZE bytes" + echo "Created at: $CREATED_AT" + echo "Expires at: $EXPIRES_AT" + echo "Download URL: $DOWNLOAD_URL" + echo "--------------------------------" + done + + # Download the first artifact + DOWNLOAD_URL=$(echo $ARTIFACTS | jq -r '.artifacts[0].archive_download_url') + + if [ ! -z "$DOWNLOAD_URL" ] && [ "$DOWNLOAD_URL" != "null" ]; then + DOWNLOAD_PATH="artifact.zip" + + echo "Downloading artifact from $DOWNLOAD_URL..." + if ! curl -L -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3+json" -o "$DOWNLOAD_PATH" "$DOWNLOAD_URL"; then + echo "Error downloading artifact." + exit 1 + fi + echo "Artifact downloaded to $DOWNLOAD_PATH" + unzip_artifact "$DOWNLOAD_PATH" "$UNZIP_FOLDER" + else + echo "No artifact download URL found." + fi + else + echo "No artifacts found." + fi +else + echo "The latest workflow run has not completed successfully." + exit 1 +fi diff --git a/visuals/exclude.json b/visuals/exclude.json new file mode 100644 index 0000000000..a177746202 --- /dev/null +++ b/visuals/exclude.json @@ -0,0 +1 @@ +["/integrations/"] diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index 1c344947ff..1243ff58c6 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -6,8 +6,12 @@ const siteUrl = "http://localhost:3000"; const sitemapPath = "build/sitemap.xml"; const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); +const excludeList = require("./excludeList.json"); -function isApiDocsPathname(pathname: string): boolean { +function isApiDocsPathname(pathname: string, excludeList: string[]): boolean { + if (excludeList.includes(pathname)) { + return false; + } // return false if the pathname does not start with /api/ if (pathname.startsWith("/api/") && !pathname.match(/^\/api\/(\d+\.\d+\.x)\//)) { return true; @@ -28,13 +32,7 @@ function screenshotPathname(pathname: string) { }); } -// test.describe("API docs screenshots", () => { -// const pathnames = extractSitemapPathnames(sitemapPath).filter(isApiDocsPathname); -// pathnames.forEach(screenshotPathname); -// }); - test.describe("API docs screenshots", () => { - const pathnames = extractSitemapPathnames(sitemapPath).filter(isApiDocsPathname); - console.log("Total pathnames: ", pathnames.length); - screenshotPathname(pathnames[0]); + const pathnames = extractSitemapPathnames(sitemapPath).filter((pathname) => isApiDocsPathname(pathname, excludeList)); + pathnames.forEach(screenshotPathname); }); diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index d6c023f554..a83d15aaf0 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -6,8 +6,13 @@ const siteUrl = "http://localhost:3000"; const sitemapPath = "build/sitemap.xml"; const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); +const excludeList = require("./excludeList.json"); + +function isVersionedDocsPathname(pathname: string, excludeList: string[]): boolean { + if (excludeList.includes(pathname)) { + return false; + } -function isVersionedDocsPathname(pathname: string): boolean { if (pathname.startsWith("/api/") || pathname.match(/\/\d+\.\d+\.x\//)) { return false; } @@ -39,13 +44,9 @@ function screenshotPathname(pathname: string) { // }); // }); -// test.describe("Docusaurus site screenshots", () => { -// const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); -// pathnames.forEach(screenshotPathname); -// }); - test.describe("API docs screenshots", () => { - const pathnames = extractSitemapPathnames(sitemapPath).filter(isVersionedDocsPathname); - console.log("Total pathnames: ", pathnames.length); - screenshotPathname(pathnames[0]); + const pathnames = extractSitemapPathnames(sitemapPath).filter((pathname) => + isVersionedDocsPathname(pathname, excludeList) + ); + pathnames.forEach(screenshotPathname); }); From 948fa5cdec2eb5495803ced78a9c653a82c03e39 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 15:08:47 -0700 Subject: [PATCH 40/58] ci: fix test path --- .github/workflows/screenshot_capture.yaml | 10 +++++----- .github/workflows/visual-comparison.yaml | 4 ++++ visuals/screenshot.api.spec.ts | 2 +- visuals/screenshot.docs.spec.ts | 2 +- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 83431ccf7c..b140bc6519 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -33,25 +33,25 @@ jobs: - name: Checkout Repository uses: actions/checkout@v4 - - name: Setup Node.js environment + - name: Setup Node.js Environment uses: actions/setup-node@v4 with: node-version: "18" cache: "npm" - - name: Install dependencies + - name: Install Dependencies run: npm ci - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Build the website + - name: Build Website run: make build - - name: Take screenshots with Playwright + - name: Take Screenshots with Playwright run: make test-visual-ci - - name: Upload screenshots + - name: Upload Screenshots uses: actions/upload-artifact@v4 id: screenshots with: diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 8f2f6b457a..0ad93976f5 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -130,6 +130,10 @@ jobs: fi done + + - name: DNS Wait + run: sleep 120 + - name: GH Pages URL run: | diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index 1243ff58c6..65ea8e5f76 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -6,7 +6,7 @@ const siteUrl = "http://localhost:3000"; const sitemapPath = "build/sitemap.xml"; const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); -const excludeList = require("./excludeList.json"); +const excludeList = require("./exclude.json"); function isApiDocsPathname(pathname: string, excludeList: string[]): boolean { if (excludeList.includes(pathname)) { diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index a83d15aaf0..56bff4786e 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -6,7 +6,7 @@ const siteUrl = "http://localhost:3000"; const sitemapPath = "build/sitemap.xml"; const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); -const excludeList = require("./excludeList.json"); +const excludeList = require("./exclude.json"); function isVersionedDocsPathname(pathname: string, excludeList: string[]): boolean { if (excludeList.includes(pathname)) { From c85428403d31e0b3786a7bd23eaab973272ab9c2 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 15:58:36 -0700 Subject: [PATCH 41/58] test: new logic --- .github/workflows/screenshot_capture.yaml | 12 ++++-------- .github/workflows/visual-comparison.yaml | 18 +++++++++--------- scripts/screenshot_artifacts.sh | 2 +- visuals/screenshot.api.spec.ts | 1 + visuals/screenshot.docs.spec.ts | 1 + 5 files changed, 16 insertions(+), 18 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index b140bc6519..d8c70f21da 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -4,15 +4,11 @@ name: Screenshot Capture # The screenshots can be used to compare the visual appearance of the website before and after a release. # The workflow Visual Comparison uses these screenshots to generate a visual comparison report. -# on: -# workflow_run: -# workflows: ["Release to Production"] -# types: [completed] -# workflow_dispatch: - on: - push: - branches: [visual-tests] + workflow_run: + workflows: ["Release to Production"] + types: [completed] + workflow_dispatch: env: diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 0ad93976f5..7ebd2cff0d 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -1,16 +1,16 @@ name: Visual Comparison -# on: -# push: -# branches: [visual-tests] +on: + push: + branches: [visual-tests] -on: - pull_request: - types: [opened, reopened, synchronize, labeled] - branches-ignore: ["version-*", "release-*"] - paths: - - 'src/**' +# on: +# pull_request: +# types: [opened, reopened, synchronize, labeled] +# branches-ignore: ["version-*", "release-*"] +# paths: +# - 'src/**' env: diff --git a/scripts/screenshot_artifacts.sh b/scripts/screenshot_artifacts.sh index 00c5b2b429..ab373fffc2 100755 --- a/scripts/screenshot_artifacts.sh +++ b/scripts/screenshot_artifacts.sh @@ -36,7 +36,7 @@ OWNER="${GITHUB_REPOSITORY_OWNER:-spectrocloud}" REPO_NAME="${GITHUB_REPOSITORY#*/}" REPO="${REPO_NAME:-librarium}" WORKFLOW_FILE="screenshot_capture.yaml" -UNZIP_FOLDER="screenshots" +UNZIP_FOLDER="../screenshots" perform_curl() { diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index 65ea8e5f76..8dd9a7345b 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -10,6 +10,7 @@ const excludeList = require("./exclude.json"); function isApiDocsPathname(pathname: string, excludeList: string[]): boolean { if (excludeList.includes(pathname)) { + console.log(`Excluding ${pathname}`); return false; } // return false if the pathname does not start with /api/ diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index 56bff4786e..5190d18bee 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -10,6 +10,7 @@ const excludeList = require("./exclude.json"); function isVersionedDocsPathname(pathname: string, excludeList: string[]): boolean { if (excludeList.includes(pathname)) { + console.log(`Excluding ${pathname}`); return false; } From 33d31183631f869235f98e1a9d7ea67ca91cfa28 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 16:17:02 -0700 Subject: [PATCH 42/58] ci: fix download path --- .github/workflows/visual-comparison.yaml | 2 +- .gitignore | 3 ++- Makefile | 4 ++-- scripts/screenshot_artifacts.sh | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 7ebd2cff0d..6a05bb2017 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -74,7 +74,7 @@ jobs: run: ./scripts/screenshot_artifacts.sh - name: Take screenshots with Playwright - run: make test-visual-ci + run: make test-visuals-ci - name: Upload Report uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index d1d71dcb29..e2d345d0f9 100644 --- a/.gitignore +++ b/.gitignore @@ -58,4 +58,5 @@ cosign.key screenshots/ tests/screenshot.spec.ts-snapshots/ test-results/ -playwright-report/ \ No newline at end of file +playwright-report/ +artifact.zip \ No newline at end of file diff --git a/Makefile b/Makefile index 4f7ecbae58..505f39ec13 100644 --- a/Makefile +++ b/Makefile @@ -75,10 +75,10 @@ api: ## Generate API docs test: ## Run Jest tests npm test -test-visual: ## Run visual regression tests +test-visuals: ## Run visual regression tests npx playwright test visuals/ --workers $(CPUS) -test-visual-ci: ## Run visual regression tests +test-visuals-ci: ## Run visual regression tests npx playwright test visuals/ --workers 4 || exit 0 ##@ Git Targets diff --git a/scripts/screenshot_artifacts.sh b/scripts/screenshot_artifacts.sh index ab373fffc2..00c5b2b429 100755 --- a/scripts/screenshot_artifacts.sh +++ b/scripts/screenshot_artifacts.sh @@ -36,7 +36,7 @@ OWNER="${GITHUB_REPOSITORY_OWNER:-spectrocloud}" REPO_NAME="${GITHUB_REPOSITORY#*/}" REPO="${REPO_NAME:-librarium}" WORKFLOW_FILE="screenshot_capture.yaml" -UNZIP_FOLDER="../screenshots" +UNZIP_FOLDER="screenshots" perform_curl() { From 0e255d0937a70fbc539f9a3a9f696155e9dde6ad Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 17:09:46 -0700 Subject: [PATCH 43/58] chore: ready --- .github/workflows/visual-comparison.yaml | 18 ++++++++---------- playwright.config.ts | 4 ++-- visuals/screenshot.api.spec.ts | 2 +- visuals/screenshot.docs.spec.ts | 2 +- 4 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 6a05bb2017..bf5a056279 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -1,16 +1,14 @@ name: Visual Comparison -on: - push: - branches: [visual-tests] - -# on: -# pull_request: -# types: [opened, reopened, synchronize, labeled] -# branches-ignore: ["version-*", "release-*"] -# paths: -# - 'src/**' +on: + pull_request: + types: [opened, reopened, synchronize, labeled] + branches-ignore: ["version-*", "release-*"] + paths: + - 'src/**' + - 'package.json' + - 'package-lock.json' env: diff --git a/playwright.config.ts b/playwright.config.ts index 6d26c3b9d2..247f730f81 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -8,10 +8,10 @@ const config: PlaywrightTestConfig = { fullyParallel: true, expect: { toMatchSnapshot: { - maxDiffPixels: 10, + maxDiffPixels: 100, }, toHaveScreenshot: { - maxDiffPixels: 10, + maxDiffPixels: 100, }, }, workers: process.env.CI ? 1 : 2, diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index 8dd9a7345b..02462f854c 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -28,7 +28,7 @@ function screenshotPathname(pathname: string) { await page.waitForFunction(WaitForDocusaurusHydration); await page.waitForLoadState("domcontentloaded"); await page.addStyleTag({ content: stylesheet }); - await page.waitForTimeout(500); // Waits for 100 milliseconds + await page.waitForTimeout(1000); // Waits for 1000 milliseconds await expect(page).toHaveScreenshot({ fullPage: true, timeout: 10000 }); }); } diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index 5190d18bee..035db10dd5 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -29,7 +29,7 @@ function screenshotPathname(pathname: string) { await page.waitForFunction(WaitForDocusaurusHydration); await page.waitForLoadState("domcontentloaded"); await page.addStyleTag({ content: stylesheet }); - await page.waitForTimeout(500); // Waits for 100 milliseconds + await page.waitForTimeout(1000); // Waits for 1000 milliseconds await expect(page).toHaveScreenshot({ fullPage: true, timeout: 10000 }); }); } From 193590213f9d37798056e34a298a71ab35a4fef5 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Sun, 31 Mar 2024 18:16:34 -0700 Subject: [PATCH 44/58] save --- .github/workflows/visual-comparison.yaml | 10 +++++++++- visuals/screenshot.css | 8 ++++++++ visuals/screenshot.docs.spec.ts | 2 +- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index bf5a056279..b52788ced0 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -71,6 +71,14 @@ jobs: - name: Download Reference Screenshots run: ./scripts/screenshot_artifacts.sh + + - name: Post Netlify progress + uses: mshick/add-pr-comment@v2 + with: + message: | + 🤖 Starting the visual tests. This will take approximately 45 minutes. + refresh-message-position: true + - name: Take screenshots with Playwright run: make test-visuals-ci @@ -138,7 +146,7 @@ jobs: echo "::notice title= Published Playwright Test Report::https://spectrocloud.github.io/librarium/$HTML_REPORT_URL_PATH" - - name: Post Netlify URL + - name: Post Report URL Comment uses: mshick/add-pr-comment@v2 with: message: | diff --git a/visuals/screenshot.css b/visuals/screenshot.css index 292a70aaf8..c6b8e78eda 100644 --- a/visuals/screenshot.css +++ b/visuals/screenshot.css @@ -41,3 +41,11 @@ Hide Cookie Consent banner #usercentrics-root { display: none !important; } + +/* +API Request Form + +*/ +.openapi-explorer__request-form { + display: none; +} diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index 035db10dd5..de50fb7771 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -45,7 +45,7 @@ function screenshotPathname(pathname: string) { // }); // }); -test.describe("API docs screenshots", () => { +test.describe("Docs screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter((pathname) => isVersionedDocsPathname(pathname, excludeList) ); From 729fdbcb2ec681d19d242925cd0a37a4ee7648a3 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 07:55:05 -0700 Subject: [PATCH 45/58] ci: more tests --- .github/workflows/screenshot_capture.yaml | 47 +++++++++- .github/workflows/visual-comparison.yaml | 103 +++++++++++++++++----- Makefile | 7 +- playwright.config.ts | 2 +- visuals/screenshot.api.spec.ts | 4 +- visuals/screenshot.docs.spec.ts | 10 +++ 6 files changed, 147 insertions(+), 26 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index d8c70f21da..7be2aba482 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -23,7 +23,8 @@ env: jobs: - visual-snapshots: + build: + name: Build runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -44,8 +45,50 @@ jobs: - name: Build Website run: make build + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: "build" + path: | + build/ + if-no-files-found: error + retention-days: 1 + + + + + visual-snapshots: + runs-on: ubuntu-latest + needs: [build] + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Node.js Environment + uses: actions/setup-node@v4 + with: + node-version: "18" + cache: "npm" + + - name: Install Dependencies + run: npm ci + + - name: Install Playwright browsers + run: npx playwright install --with-deps chromium + + - name: Download Build + uses: actions/download-artifact@v4 + with: + name: build + path: build + - name: Take Screenshots with Playwright - run: make test-visual-ci + run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} - name: Upload Screenshots uses: actions/upload-artifact@v4 diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index b52788ced0..aba0f2576b 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -45,10 +45,54 @@ jobs: # As of 8/8/2022 there is now way to enforce this beahvior in GitHub Actions CI. - run: exit 0 + + build: + name: Build + needs: [run-ci] + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Setup Node.js Environment + uses: actions/setup-node@v4 + with: + node-version: "18" + cache: "npm" + + - name: Post PR Comment + uses: mshick/add-pr-comment@v2 + with: + message: | + 🤖 Starting the visual tests. This will take approximately an hour. + refresh-message-position: true + + - name: Install Dependencies + run: npm ci + + - name: Build Website + run: make build + + - name: Upload Build + uses: actions/upload-artifact@v4 + with: + name: "build" + path: | + build/ + if-no-files-found: error + retention-days: 1 + + + take-screenshots: name: Visual Comparison - needs: [run-ci] + needs: [run-ci, build] runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4] + shardTotal: [4] steps: - name: Check out repository code uses: actions/checkout@v4 @@ -65,37 +109,56 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Build the website - run: make build + - name: Download Build + uses: actions/download-artifact@v4 + with: + name: build + path: build - name: Download Reference Screenshots run: ./scripts/screenshot_artifacts.sh - - - name: Post Netlify progress - uses: mshick/add-pr-comment@v2 - with: - message: | - 🤖 Starting the visual tests. This will take approximately 45 minutes. - refresh-message-position: true - - name: Take screenshots with Playwright - run: make test-visuals-ci + run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} - - name: Upload Report + - name: Upload blob report to GitHub Actions Artifacts uses: actions/upload-artifact@v4 - id: report with: - name: "report" - path: | - playwright-report/ - if-no-files-found: error - retention-days: 1 + name: blob-report-${{ matrix.shardIndex }} + path: blob-report + retention-days: 1 + + + merge-reports: + name: Merge Reports + needs: [take-screenshots] + runs-on: ubuntu-latest + steps: + + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v4 + with: + path: all-blob-reports + pattern: blob-report-* + merge-multiple: true + + - name: Merge into HTML Report + run: npx playwright merge-reports --reporter html ./all-blob-reports + + - name: Upload Report + uses: actions/upload-artifact@v4 + id: report + with: + name: "report" + path: | + playwright-report/ + if-no-files-found: error + retention-days: 1 publish_report: name: Publish HTML Report - needs: [run-ci, take-screenshots] + needs: [run-ci, take-screenshots, merge-reports] runs-on: ubuntu-latest continue-on-error: true steps: diff --git a/Makefile b/Makefile index 505f39ec13..c7f65d48ec 100644 --- a/Makefile +++ b/Makefile @@ -76,10 +76,13 @@ test: ## Run Jest tests npm test test-visuals: ## Run visual regression tests - npx playwright test visuals/ --workers $(CPUS) + npx playwright test visuals/ test-visuals-ci: ## Run visual regression tests - npx playwright test visuals/ --workers 4 || exit 0 + npx playwright test --shard=1/4 + npx playwright test --shard=2/4 + npx playwright test --shard=3/4 + npx playwright test --shard=4/4 ##@ Git Targets diff --git a/playwright.config.ts b/playwright.config.ts index 247f730f81..b9ca554b81 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -6,6 +6,7 @@ const config: PlaywrightTestConfig = { snapshotDir: "screenshots/", testDir: "visuals", fullyParallel: true, + retries: 1, expect: { toMatchSnapshot: { maxDiffPixels: 100, @@ -14,7 +15,6 @@ const config: PlaywrightTestConfig = { maxDiffPixels: 100, }, }, - workers: process.env.CI ? 1 : 2, reporter: [["html", { open: "never" }]], webServer: { command: "npm run serve", diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index 02462f854c..e8dd1dcf36 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -1,13 +1,15 @@ import * as fs from "fs"; import { test, expect } from "@playwright/test"; import { extractSitemapPathnames, WaitForDocusaurusHydration } from "./utils"; -// Constants: + const siteUrl = "http://localhost:3000"; const sitemapPath = "build/sitemap.xml"; const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); const excludeList = require("./exclude.json"); +test.describe.configure({ mode: "parallel" }); + function isApiDocsPathname(pathname: string, excludeList: string[]): boolean { if (excludeList.includes(pathname)) { console.log(`Excluding ${pathname}`); diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index de50fb7771..78dd60bf1b 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -8,6 +8,8 @@ const stylesheetPath = "visuals/screenshot.css"; const stylesheet = fs.readFileSync(stylesheetPath).toString(); const excludeList = require("./exclude.json"); +test.describe.configure({ mode: "parallel" }); + function isVersionedDocsPathname(pathname: string, excludeList: string[]): boolean { if (excludeList.includes(pathname)) { console.log(`Excluding ${pathname}`); @@ -21,6 +23,14 @@ function isVersionedDocsPathname(pathname: string, excludeList: string[]): boole return true; } +// function chunkArray(array: string[], parts: number): string[][] { +// let result: string[][] = []; +// for (let i = parts; i > 0; i--) { +// result.push(array.splice(0, Math.ceil(array.length / i))); +// } +// return result; +// } + function screenshotPathname(pathname: string) { test(`pathname ${pathname}`, async ({ page }) => { console.log(`Taking screenshot of ${pathname}`); From 12b77e63acc6903d0f8db02e523cfc8d12970715 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 08:41:57 -0700 Subject: [PATCH 46/58] ci: fix screenshots --- .github/workflows/visual-comparison.yaml | 2 +- playwright.config.ts | 1 + visuals/screenshot.api.spec.ts | 4 +++- visuals/screenshot.docs.spec.ts | 23 +++-------------------- 4 files changed, 8 insertions(+), 22 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index aba0f2576b..31b3e28f81 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -119,7 +119,7 @@ jobs: run: ./scripts/screenshot_artifacts.sh - name: Take screenshots with Playwright - run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0 - name: Upload blob report to GitHub Actions Artifacts uses: actions/upload-artifact@v4 diff --git a/playwright.config.ts b/playwright.config.ts index b9ca554b81..26dfa81892 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -14,6 +14,7 @@ const config: PlaywrightTestConfig = { toHaveScreenshot: { maxDiffPixels: 100, }, + timeout: 30000, }, reporter: [["html", { open: "never" }]], webServer: { diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index e8dd1dcf36..3481cbec45 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -12,7 +12,6 @@ test.describe.configure({ mode: "parallel" }); function isApiDocsPathname(pathname: string, excludeList: string[]): boolean { if (excludeList.includes(pathname)) { - console.log(`Excluding ${pathname}`); return false; } // return false if the pathname does not start with /api/ @@ -37,5 +36,8 @@ function screenshotPathname(pathname: string) { test.describe("API docs screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter((pathname) => isApiDocsPathname(pathname, excludeList)); + console.log(`Taking screenshots of ${pathnames.length} API docs pages`); + console.log("Excluded pages: ", excludeList); + pathnames.forEach(screenshotPathname); }); diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index 78dd60bf1b..ae8d0b2763 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -23,17 +23,8 @@ function isVersionedDocsPathname(pathname: string, excludeList: string[]): boole return true; } -// function chunkArray(array: string[], parts: number): string[][] { -// let result: string[][] = []; -// for (let i = parts; i > 0; i--) { -// result.push(array.splice(0, Math.ceil(array.length / i))); -// } -// return result; -// } - function screenshotPathname(pathname: string) { test(`pathname ${pathname}`, async ({ page }) => { - console.log(`Taking screenshot of ${pathname}`); const url = siteUrl + pathname; await page.goto(url); await page.waitForFunction(WaitForDocusaurusHydration); @@ -44,20 +35,12 @@ function screenshotPathname(pathname: string) { }); } -// test.describe("cookie-banner is visible", () => { -// test("cookie-banner is visible", async ({ page }) => { -// await page.goto(siteUrl); -// await page.waitForFunction(WaitForDocusaurusHydration); -// await page.waitForLoadState("domcontentloaded"); -// await page.addStyleTag({ content: stylesheet }); -// await expect(page).toHaveScreenshot({ fullPage: true }); -// await expect(page.getByTestId("#usercentrics-root")).toBeVisible(); -// }); -// }); - test.describe("Docs screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter((pathname) => isVersionedDocsPathname(pathname, excludeList) ); + console.log(`Taking screenshots of ${pathnames.length} Docs pages`); + console.log("Excluded pages: ", excludeList); + pathnames.forEach(screenshotPathname); }); From b240e0695621092ff91e7b63acf55f0f983ac63c Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 09:43:51 -0700 Subject: [PATCH 47/58] ci: more changes --- .github/workflows/screenshot_capture.yaml | 16 ++++++++++------ .github/workflows/visual-comparison.yaml | 14 +++++++++----- visuals/screenshot.api.spec.ts | 8 +++++--- visuals/screenshot.docs.spec.ts | 8 ++++++-- 4 files changed, 30 insertions(+), 16 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 7be2aba482..320cc063c9 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -81,14 +81,18 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Download Build - uses: actions/download-artifact@v4 - with: - name: build - path: build + - name: retry action + uses: Wandalen/wretry.action@v3 + with: + attempt_limit: 3 + action: actions/download-artifact@v4 + with: | + name: build + path: build + attempt_delay: 60000 # 1 minute - name: Take Screenshots with Playwright - run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} + run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0 - name: Upload Screenshots uses: actions/upload-artifact@v4 diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 31b3e28f81..98b7ecb0b9 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -109,11 +109,15 @@ jobs: - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: Download Build - uses: actions/download-artifact@v4 - with: - name: build - path: build + - name: retry action + uses: Wandalen/wretry.action@v3 + with: + attempt_limit: 3 + action: actions/download-artifact@v4 + with: | + name: build + path: build + attempt_delay: 60000 # 1 minute - name: Download Reference Screenshots run: ./scripts/screenshot_artifacts.sh diff --git a/visuals/screenshot.api.spec.ts b/visuals/screenshot.api.spec.ts index 3481cbec45..de697b9c94 100644 --- a/visuals/screenshot.api.spec.ts +++ b/visuals/screenshot.api.spec.ts @@ -21,6 +21,11 @@ function isApiDocsPathname(pathname: string, excludeList: string[]): boolean { return false; } +test.beforeAll(async () => { + console.log("Excluded pages: ", excludeList); + console.log("Total pages: ", extractSitemapPathnames(sitemapPath).length); +}); + function screenshotPathname(pathname: string) { test(`pathname ${pathname}`, async ({ page }) => { console.log(`Taking screenshot of ${pathname}`); @@ -36,8 +41,5 @@ function screenshotPathname(pathname: string) { test.describe("API docs screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter((pathname) => isApiDocsPathname(pathname, excludeList)); - console.log(`Taking screenshots of ${pathnames.length} API docs pages`); - console.log("Excluded pages: ", excludeList); - pathnames.forEach(screenshotPathname); }); diff --git a/visuals/screenshot.docs.spec.ts b/visuals/screenshot.docs.spec.ts index ae8d0b2763..9588901edc 100644 --- a/visuals/screenshot.docs.spec.ts +++ b/visuals/screenshot.docs.spec.ts @@ -23,8 +23,14 @@ function isVersionedDocsPathname(pathname: string, excludeList: string[]): boole return true; } +test.beforeAll(async () => { + console.log("Excluded pages: ", excludeList); + console.log("Total pages: ", extractSitemapPathnames(sitemapPath).length); +}); + function screenshotPathname(pathname: string) { test(`pathname ${pathname}`, async ({ page }) => { + console.log(`Taking screenshot of ${pathname}`); const url = siteUrl + pathname; await page.goto(url); await page.waitForFunction(WaitForDocusaurusHydration); @@ -39,8 +45,6 @@ test.describe("Docs screenshots", () => { const pathnames = extractSitemapPathnames(sitemapPath).filter((pathname) => isVersionedDocsPathname(pathname, excludeList) ); - console.log(`Taking screenshots of ${pathnames.length} Docs pages`); - console.log("Excluded pages: ", excludeList); pathnames.forEach(screenshotPathname); }); From c1f61566d60e71f00e4c9e9fdb881930abced299 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 10:03:43 -0700 Subject: [PATCH 48/58] chore: more tests --- .github/workflows/visual-comparison.yaml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 98b7ecb0b9..7288a93212 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -138,6 +138,17 @@ jobs: needs: [take-screenshots] runs-on: ubuntu-latest steps: + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Setup Node.js environment + uses: actions/setup-node@v4 + with: + node-version: "18" + cache: "npm" + + - name: Install dependencies + run: npm ci - name: Download blob reports from GitHub Actions Artifacts uses: actions/download-artifact@v4 From 425c5c4375cc041f157c9a60eeec2958b5be2f53 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 11:27:55 -0700 Subject: [PATCH 49/58] ci: more tests --- .github/workflows/visual-comparison.yaml | 2 +- playwright.config.ts | 3 ++- scripts/screenshot_artifacts.sh | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 7288a93212..39fa571fd9 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -120,7 +120,7 @@ jobs: attempt_delay: 60000 # 1 minute - name: Download Reference Screenshots - run: ./scripts/screenshot_artifacts.sh + run: ./scripts/screenshot_artifacts.sh && ls -ll - name: Take screenshots with Playwright run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0 diff --git a/playwright.config.ts b/playwright.config.ts index 26dfa81892..90aa0b7b8f 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -7,6 +7,7 @@ const config: PlaywrightTestConfig = { testDir: "visuals", fullyParallel: true, retries: 1, + expect: { toMatchSnapshot: { maxDiffPixels: 100, @@ -16,7 +17,7 @@ const config: PlaywrightTestConfig = { }, timeout: 30000, }, - reporter: [["html", { open: "never" }]], + reporter: process.env.CI ? "blob" : "html", webServer: { command: "npm run serve", url: "http://127.0.0.1:3000", diff --git a/scripts/screenshot_artifacts.sh b/scripts/screenshot_artifacts.sh index 00c5b2b429..868a2ccc4e 100755 --- a/scripts/screenshot_artifacts.sh +++ b/scripts/screenshot_artifacts.sh @@ -13,6 +13,8 @@ set -e +DESTINATION_FOLDER=$1 + # Function to unzip the artifact unzip_artifact() { local zip_file=$1 @@ -36,7 +38,7 @@ OWNER="${GITHUB_REPOSITORY_OWNER:-spectrocloud}" REPO_NAME="${GITHUB_REPOSITORY#*/}" REPO="${REPO_NAME:-librarium}" WORKFLOW_FILE="screenshot_capture.yaml" -UNZIP_FOLDER="screenshots" +UNZIP_FOLDER="${DESTINATION_FOLDER:-screenshots}" perform_curl() { From fd0a3e7db6cfd4afc82e3a09be8375aa70a11af3 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 11:47:07 -0700 Subject: [PATCH 50/58] chore: restart --- .github/workflows/screenshot_capture.yaml | 8 ++++++-- .github/workflows/visual-comparison.yaml | 12 ++++++++---- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 320cc063c9..d9606cae6e 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -75,8 +75,12 @@ jobs: node-version: "18" cache: "npm" - - name: Install Dependencies - run: npm ci + - name: Download Build + uses: Wandalen/wretry.action@v3 + with: + command: npm ci + attempt_limit: 3 + attempt_delay: 60000 # 1 minute - name: Install Playwright browsers run: npx playwright install --with-deps chromium diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 39fa571fd9..7dd0421c19 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -103,13 +103,17 @@ jobs: node-version: "18" cache: "npm" - - name: Install dependencies - run: npm ci + - name: Download Build + uses: Wandalen/wretry.action@v3 + with: + command: npm ci + attempt_limit: 3 + attempt_delay: 60000 # 1 minute - name: Install Playwright browsers run: npx playwright install --with-deps chromium - - name: retry action + - name: Download Build uses: Wandalen/wretry.action@v3 with: attempt_limit: 3 @@ -120,7 +124,7 @@ jobs: attempt_delay: 60000 # 1 minute - name: Download Reference Screenshots - run: ./scripts/screenshot_artifacts.sh && ls -ll + run: ./scripts/screenshot_artifacts.sh ./screenshots && ls -ll - name: Take screenshots with Playwright run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0 From bd3620cf9f6ad0eaf9eba341b3f285153815d4f7 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 13:07:10 -0700 Subject: [PATCH 51/58] chore: update --- .github/workflows/screenshot_capture.yaml | 14 ++++++++++---- .github/workflows/visual-comparison.yaml | 4 ++-- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index d9606cae6e..65babedd38 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -4,11 +4,17 @@ name: Screenshot Capture # The screenshots can be used to compare the visual appearance of the website before and after a release. # The workflow Visual Comparison uses these screenshots to generate a visual comparison report. +# on: +# workflow_run: +# workflows: ["Release to Production"] +# types: [completed] +# workflow_dispatch: + + on: - workflow_run: - workflows: ["Release to Production"] - types: [completed] - workflow_dispatch: + push: + branches: + - visual-tests env: diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 7dd0421c19..f27dbe5cde 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -28,7 +28,7 @@ env: concurrency: group: "visual-comparison-${{ github.workflow }}" - cancel-in-progress: false + cancel-in-progress: true jobs: @@ -124,7 +124,7 @@ jobs: attempt_delay: 60000 # 1 minute - name: Download Reference Screenshots - run: ./scripts/screenshot_artifacts.sh ./screenshots && ls -ll + run: ./scripts/screenshot_artifacts.sh ./screenshots - name: Take screenshots with Playwright run: npx playwright test --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} || exit 0 From 39905dc94cef842bbea0091a970e9f7998f6ff60 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 13:26:17 -0700 Subject: [PATCH 52/58] ci: removed Appzi from builds --- .github/workflows/screenshot_capture.yaml | 1 - .github/workflows/visual-comparison.yaml | 19 +++++++++++++------ 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 65babedd38..2ef8343208 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -19,7 +19,6 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }} MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }} FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }} ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index f27dbe5cde..31711e2560 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -16,7 +16,6 @@ env: AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }} AWS_DEFAULT_REGION: us-east-1 - APPZI_TOKEN: ${{ secrets.APPZI_TOKEN }} MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }} FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }} ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} @@ -68,7 +67,11 @@ jobs: refresh-message-position: true - name: Install Dependencies - run: npm ci + uses: Wandalen/wretry.action@v3 + with: + command: npm ci + attempt_limit: 3 + attempt_delay: 60000 # 1 minute - name: Build Website run: make build @@ -103,7 +106,7 @@ jobs: node-version: "18" cache: "npm" - - name: Download Build + - name: Install Dependencies uses: Wandalen/wretry.action@v3 with: command: npm ci @@ -151,8 +154,12 @@ jobs: node-version: "18" cache: "npm" - - name: Install dependencies - run: npm ci + - name: Install Dependencies + uses: Wandalen/wretry.action@v3 + with: + command: npm ci + attempt_limit: 3 + attempt_delay: 60000 # 1 minute - name: Download blob reports from GitHub Actions Artifacts uses: actions/download-artifact@v4 @@ -233,7 +240,7 @@ jobs: with: message: | 📋 Visual Report for ${{ github.ref_name }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready attempt at - https://spectrocloud.github.io/$GITHUB_REPOSITORY_OWNER/$HTML_REPORT_URL_PATH + https://spectrocloud.github.io/${{env.GITHUB_REPOSITORY_OWNER}}/${{env.HTML_REPORT_URL_PATH}} message-failure: | 👎 Uh oh! Unable to publish Visual Report URL. refresh-message-position: true From f0d24d3eb356e9dabece803fc727058c4cc627d6 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 13:43:59 -0700 Subject: [PATCH 53/58] chore: save --- .github/workflows/screenshot_capture.yaml | 35 +++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 2ef8343208..dc54dd2e6d 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -107,8 +107,39 @@ jobs: uses: actions/upload-artifact@v4 id: screenshots with: - name: "snapshots" + name: snapshots-${{ matrix.shardIndex }} path: | screenshots/ if-no-files-found: error - retention-days: 3 \ No newline at end of file + retention-days: 1 + + merge-snapshots: + name: Merge Snapshots + runs-on: ubuntu-latest + needs: [visual-snapshots] + steps: + + - name: Check out repository code + uses: actions/checkout@v4 + + - name: Download blob reports from GitHub Actions Artifacts + uses: actions/download-artifact@v4 + with: + path: snapshots + pattern: snapshots-* + merge-multiple: true + + - name: Merge Snapshots + run: | + ls -ll + ls -ll snapshots + + - name: Upload Screenshots + uses: actions/upload-artifact@v4 + id: screenshots + with: + name: "snapshots" + path: | + snapshots/ + if-no-files-found: error + retention-days: 3 From bf3f210c192093ad1e0e7c98a27f70e192af8f92 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 15:18:42 -0700 Subject: [PATCH 54/58] ci: new test --- .github/workflows/screenshot_capture.yaml | 19 ++++++++----------- .github/workflows/visual-comparison.yaml | 6 ++++-- playwright.config.ts | 1 - 3 files changed, 12 insertions(+), 14 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index dc54dd2e6d..70e2375dce 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -80,7 +80,7 @@ jobs: node-version: "18" cache: "npm" - - name: Download Build + - name: Install Dependencies uses: Wandalen/wretry.action@v3 with: command: npm ci @@ -107,39 +107,36 @@ jobs: uses: actions/upload-artifact@v4 id: screenshots with: - name: snapshots-${{ matrix.shardIndex }} + name: screenshots-${{ matrix.shardIndex }} path: | screenshots/ if-no-files-found: error retention-days: 1 merge-snapshots: - name: Merge Snapshots + name: Merge Screenshots runs-on: ubuntu-latest needs: [visual-snapshots] steps: - - name: Check out repository code - uses: actions/checkout@v4 - - name: Download blob reports from GitHub Actions Artifacts uses: actions/download-artifact@v4 with: - path: snapshots - pattern: snapshots-* + path: screenshots + pattern: screenshots-* merge-multiple: true - name: Merge Snapshots run: | ls -ll - ls -ll snapshots + ls -ll screenshots - name: Upload Screenshots uses: actions/upload-artifact@v4 id: screenshots with: - name: "snapshots" + name: "screenshots" path: | - snapshots/ + screenshots/ if-no-files-found: error retention-days: 3 diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 31711e2560..145dc8db66 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -207,7 +207,7 @@ jobs: path: ${{ env.HTML_REPORT_URL_PATH }} - name: Push HTML Report - timeout-minutes: 3 + timeout-minutes: 5 run: | git add . git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt: ${{ github.run_attempt }})" @@ -240,7 +240,9 @@ jobs: with: message: | 📋 Visual Report for ${{ github.ref_name }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready attempt at - https://spectrocloud.github.io/${{env.GITHUB_REPOSITORY_OWNER}}/${{env.HTML_REPORT_URL_PATH}} + https://spectrocloud.github.io/${{ github.GITHUB_REPOSITORY_OWNER}}/${{env.HTML_REPORT_URL_PATH}} + + 💡 You may have to wait for DNS to resolve or the GitHub Pages job to complete. You can view the progress of the GitHub Pages job [here](https://github.com/spectrocloud/librarium/actions/workflows/pages/pages-build-deployment). message-failure: | 👎 Uh oh! Unable to publish Visual Report URL. refresh-message-position: true diff --git a/playwright.config.ts b/playwright.config.ts index 90aa0b7b8f..a63f9ec0a1 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -7,7 +7,6 @@ const config: PlaywrightTestConfig = { testDir: "visuals", fullyParallel: true, retries: 1, - expect: { toMatchSnapshot: { maxDiffPixels: 100, From 3a188f4689c69ae56a02166a8202772b676a075f Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 17:02:10 -0700 Subject: [PATCH 55/58] save --- .github/workflows/screenshot_capture.yaml | 6 +++--- .github/workflows/visual-comparison.yaml | 8 ++++---- Makefile | 4 ++++ playwright.config.ts | 2 -- 4 files changed, 11 insertions(+), 9 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index 70e2375dce..d918a65747 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -28,8 +28,8 @@ env: jobs: - build: - name: Build + create-assets: + name: asset-builds runs-on: ubuntu-latest steps: - name: Checkout Repository @@ -64,7 +64,7 @@ jobs: visual-snapshots: runs-on: ubuntu-latest - needs: [build] + needs: [create-assets] strategy: fail-fast: false matrix: diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 145dc8db66..3529c8267e 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -45,8 +45,8 @@ jobs: - run: exit 0 - build: - name: Build + create-assets: + name: create-assets needs: [run-ci] runs-on: ubuntu-latest steps: @@ -89,7 +89,7 @@ jobs: take-screenshots: name: Visual Comparison - needs: [run-ci, build] + needs: [run-ci, create-assets] runs-on: ubuntu-latest strategy: fail-fast: false @@ -240,7 +240,7 @@ jobs: with: message: | 📋 Visual Report for ${{ github.ref_name }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready attempt at - https://spectrocloud.github.io/${{ github.GITHUB_REPOSITORY_OWNER}}/${{env.HTML_REPORT_URL_PATH}} + https://spectrocloud.github.io/librarium/${{env.HTML_REPORT_URL_PATH}} 💡 You may have to wait for DNS to resolve or the GitHub Pages job to complete. You can view the progress of the GitHub Pages job [here](https://github.com/spectrocloud/librarium/actions/workflows/pages/pages-build-deployment). message-failure: | diff --git a/Makefile b/Makefile index c7f65d48ec..4f879211ca 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,10 @@ test-visuals-ci: ## Run visual regression tests npx playwright test --shard=3/4 npx playwright test --shard=4/4 + +view-visual-report: ## View visual regression test report + npx playwright show-report + ##@ Git Targets commit: ## Add a Git commit. Usage: make commit MESSAGE="" diff --git a/playwright.config.ts b/playwright.config.ts index a63f9ec0a1..754d40d085 100644 --- a/playwright.config.ts +++ b/playwright.config.ts @@ -1,8 +1,6 @@ -// playwright.config.ts import { PlaywrightTestConfig } from "@playwright/test"; const config: PlaywrightTestConfig = { - // Other config options... snapshotDir: "screenshots/", testDir: "visuals", fullyParallel: true, From b05d5c216e6845726b9194717ac028b2efe594cc Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Mon, 1 Apr 2024 17:10:31 -0700 Subject: [PATCH 56/58] ci: updated artifacts logic --- scripts/screenshot_artifacts.sh | 34 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 21 deletions(-) diff --git a/scripts/screenshot_artifacts.sh b/scripts/screenshot_artifacts.sh index 868a2ccc4e..0824805494 100755 --- a/scripts/screenshot_artifacts.sh +++ b/scripts/screenshot_artifacts.sh @@ -10,7 +10,6 @@ # Otherwise the artifacts cannot be downloaded. #################################################################################################### - set -e DESTINATION_FOLDER=$1 @@ -33,14 +32,12 @@ if [ -z "$GITHUB_TOKEN" ]; then exit 1 fi - OWNER="${GITHUB_REPOSITORY_OWNER:-spectrocloud}" REPO_NAME="${GITHUB_REPOSITORY#*/}" REPO="${REPO_NAME:-librarium}" WORKFLOW_FILE="screenshot_capture.yaml" UNZIP_FOLDER="${DESTINATION_FOLDER:-screenshots}" - perform_curl() { local url=$1 response=$(curl -s -f -w "%{http_code}" -o temp_response.txt -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3+json" "$url") @@ -65,52 +62,47 @@ ARTIFACTS_URL=$(echo $LATEST_RUN | jq -r '.artifacts_url') # Check if the status is "completed" and the conclusion is "success" if [[ "$STATUS" == "completed" && "$CONCLUSION" == "success" ]]; then - echo "The latest workflow run is completed successfully." + echo "The latest workflow run completed successfully ✅" # Fetch artifacts details ARTIFACTS=$(perform_curl "$ARTIFACTS_URL") - # Parse and display artifact information - ARTIFACTS_COUNT=$(echo $ARTIFACTS | jq '.total_count') + # Display total count of artifacts + ARTIFACTS_COUNT=$(echo "$ARTIFACTS" | jq '.total_count') echo "Found $ARTIFACTS_COUNT artifact(s)." - + if [[ "$ARTIFACTS_COUNT" -gt 0 ]]; then # Loop through each artifact and print details echo $ARTIFACTS | jq -c '.artifacts[]' | while read -r ARTIFACT; do NAME=$(echo $ARTIFACT | jq -r '.name') - DOWNLOAD_URL=$(echo $ARTIFACT | jq -r '.archive_download_url') SIZE=$(echo $ARTIFACT | jq -r '.size_in_bytes') CREATED_AT=$(echo $ARTIFACT | jq -r '.created_at') EXPIRES_AT=$(echo $ARTIFACT | jq -r '.expires_at') - echo "Artifact: $NAME" echo "Size: $SIZE bytes" echo "Created at: $CREATED_AT" echo "Expires at: $EXPIRES_AT" - echo "Download URL: $DOWNLOAD_URL" echo "--------------------------------" done - # Download the first artifact - DOWNLOAD_URL=$(echo $ARTIFACTS | jq -r '.artifacts[0].archive_download_url') - - if [ ! -z "$DOWNLOAD_URL" ] && [ "$DOWNLOAD_URL" != "null" ]; then + # Specifically download the "screenshots" artifact + DOWNLOAD_URL=$(echo "$ARTIFACTS" | jq -r '.artifacts[] | select(.name == "screenshots") | .archive_download_url') + if [[ -n "$DOWNLOAD_URL" && "$DOWNLOAD_URL" != "null" ]]; then DOWNLOAD_PATH="artifact.zip" - - echo "Downloading artifact from $DOWNLOAD_URL..." + echo "Downloading 'screenshots' artifact from $DOWNLOAD_URL..." if ! curl -L -H "Authorization: token $GITHUB_TOKEN" -H "Accept: application/vnd.github.v3+json" -o "$DOWNLOAD_PATH" "$DOWNLOAD_URL"; then - echo "Error downloading artifact." + echo "Error downloading 'screenshots' artifact." exit 1 fi - echo "Artifact downloaded to $DOWNLOAD_PATH" + echo "'screenshots' artifact downloaded to $DOWNLOAD_PATH" unzip_artifact "$DOWNLOAD_PATH" "$UNZIP_FOLDER" else - echo "No artifact download URL found." + echo "No 'screenshots' artifact download URL found ⛔" fi else - echo "No artifacts found." + echo "No artifacts found ⛔" fi else - echo "The latest workflow run has not completed successfully." + echo "The latest workflow run has not completed successfully ❌" exit 1 fi From 19bb87eed512f26b8a383cecc1ff56e6e12b61e0 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Tue, 2 Apr 2024 08:25:39 -0700 Subject: [PATCH 57/58] chore: verify self-hosted runner --- .github/workflows/clean-up-report.yaml | 2 +- .github/workflows/screenshot_capture.yaml | 14 ++++++-------- .github/workflows/visual-comparison.yaml | 14 +++++++++----- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/.github/workflows/clean-up-report.yaml b/.github/workflows/clean-up-report.yaml index 466ada9348..a81244318b 100644 --- a/.github/workflows/clean-up-report.yaml +++ b/.github/workflows/clean-up-report.yaml @@ -1,4 +1,4 @@ -name: Delete +name: Delete Visual Tests Reports on: delete: diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index d918a65747..bb125b5c8c 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -30,7 +30,8 @@ jobs: create-assets: name: asset-builds - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -63,7 +64,8 @@ jobs: visual-snapshots: - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] needs: [create-assets] strategy: fail-fast: false @@ -115,7 +117,8 @@ jobs: merge-snapshots: name: Merge Screenshots - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] needs: [visual-snapshots] steps: @@ -126,11 +129,6 @@ jobs: pattern: screenshots-* merge-multiple: true - - name: Merge Snapshots - run: | - ls -ll - ls -ll screenshots - - name: Upload Screenshots uses: actions/upload-artifact@v4 id: screenshots diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index 3529c8267e..a44964291c 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -48,7 +48,8 @@ jobs: create-assets: name: create-assets needs: [run-ci] - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -90,7 +91,8 @@ jobs: take-screenshots: name: Visual Comparison needs: [run-ci, create-assets] - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] strategy: fail-fast: false matrix: @@ -143,7 +145,8 @@ jobs: merge-reports: name: Merge Reports needs: [take-screenshots] - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] steps: - name: Check out repository code uses: actions/checkout@v4 @@ -185,7 +188,8 @@ jobs: publish_report: name: Publish HTML Report needs: [run-ci, take-screenshots, merge-reports] - runs-on: ubuntu-latest + # runs-on: ubuntu-latest + runs-on: ["self-hosted", "linux", "x64", "vcenter3"] continue-on-error: true steps: @@ -239,7 +243,7 @@ jobs: uses: mshick/add-pr-comment@v2 with: message: | - 📋 Visual Report for ${{ github.ref_name }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready attempt at + 📋 Visual Report for PR ${{ github.ref_name }} with CI run ${{ github.run_id }} and attempt ${{ github.run_attempt }} is ready at https://spectrocloud.github.io/librarium/${{env.HTML_REPORT_URL_PATH}} 💡 You may have to wait for DNS to resolve or the GitHub Pages job to complete. You can view the progress of the GitHub Pages job [here](https://github.com/spectrocloud/librarium/actions/workflows/pages/pages-build-deployment). From 4c49175a36409afac011fcfb544682f84ce8ea51 Mon Sep 17 00:00:00 2001 From: Karl Cardenas Date: Tue, 2 Apr 2024 12:15:50 -0700 Subject: [PATCH 58/58] chore: ready for merge --- .github/workflows/screenshot_capture.yaml | 24 ++++++++--------------- .github/workflows/visual-comparison.yaml | 16 ++++----------- 2 files changed, 12 insertions(+), 28 deletions(-) diff --git a/.github/workflows/screenshot_capture.yaml b/.github/workflows/screenshot_capture.yaml index bb125b5c8c..e138a540c0 100644 --- a/.github/workflows/screenshot_capture.yaml +++ b/.github/workflows/screenshot_capture.yaml @@ -4,17 +4,12 @@ name: Screenshot Capture # The screenshots can be used to compare the visual appearance of the website before and after a release. # The workflow Visual Comparison uses these screenshots to generate a visual comparison report. -# on: -# workflow_run: -# workflows: ["Release to Production"] -# types: [completed] -# workflow_dispatch: - - on: - push: - branches: - - visual-tests + workflow_run: + workflows: ["Release to Production"] + types: [completed] + workflow_dispatch: + env: @@ -30,8 +25,7 @@ jobs: create-assets: name: asset-builds - # runs-on: ubuntu-latest - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -64,8 +58,7 @@ jobs: visual-snapshots: - # runs-on: ubuntu-latest - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest needs: [create-assets] strategy: fail-fast: false @@ -117,8 +110,7 @@ jobs: merge-snapshots: name: Merge Screenshots - # runs-on: ubuntu-latest - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest needs: [visual-snapshots] steps: diff --git a/.github/workflows/visual-comparison.yaml b/.github/workflows/visual-comparison.yaml index a44964291c..b95efa2c3d 100644 --- a/.github/workflows/visual-comparison.yaml +++ b/.github/workflows/visual-comparison.yaml @@ -13,9 +13,6 @@ on: env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 MENDABLE_API_KEY: ${{ secrets.MENDABLE_API_KEY }} FULLSTORY_ORGID: ${{ secrets.FULLSTORY_ORGID }} ALGOLIA_ADMIN_KEY: ${{ secrets.ALGOLIA_ADMIN_KEY }} @@ -32,7 +29,6 @@ concurrency: jobs: run-ci: - # runs-on: ubuntu-latest runs-on: ubuntu-latest defaults: run: @@ -48,8 +44,7 @@ jobs: create-assets: name: create-assets needs: [run-ci] - # runs-on: ubuntu-latest - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest steps: - name: Checkout Repository uses: actions/checkout@v4 @@ -91,8 +86,7 @@ jobs: take-screenshots: name: Visual Comparison needs: [run-ci, create-assets] - # runs-on: ubuntu-latest - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest strategy: fail-fast: false matrix: @@ -145,8 +139,7 @@ jobs: merge-reports: name: Merge Reports needs: [take-screenshots] - # runs-on: ubuntu-latest - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest steps: - name: Check out repository code uses: actions/checkout@v4 @@ -188,8 +181,7 @@ jobs: publish_report: name: Publish HTML Report needs: [run-ci, take-screenshots, merge-reports] - # runs-on: ubuntu-latest - runs-on: ["self-hosted", "linux", "x64", "vcenter3"] + runs-on: ubuntu-latest continue-on-error: true steps: