Verify BC breakages on pull requests, by comparing PR against base branch

.eslintrc.json

@@ -22,7 +22,7 @@
         "no-unused-vars": "off",
         "@typescript-eslint/no-unused-vars": "error",
         "node/no-unpublished-import": ["error", {
-            "allowModules": ["@cfworker/json-schema"]
+            "allowModules": ["@cfworker/json-schema", "dotenv"]
         }],
         "no-process-exit": "off",
         "no-sync": "off",

.github/workflows/continuous-integration.yml

@@ -84,7 +84,15 @@ jobs:
         id: matrix_generation
         env:
           PROJECT_NAME_TO_TEST: ${{ matrix.projectName }}
-        run: cd tests/${PROJECT_NAME_TO_TEST} && docker run -i --entrypoint "/action/main.js" -v $(realpath .):/github/workspace -w=/github/workspace ${TEST_TAG} $(test -r diff && cat diff || echo -n "")
+        run: |
+          cd tests/${PROJECT_NAME_TO_TEST} && \
+          docker run \
+            -i \
+            --entrypoint "/action/main.js" \
+            -v $(realpath .):/github/workspace \
+            --env-file=test.env \
+            -w=/github/workspace \
+            ${TEST_TAG} $(test -r diff && cat diff || echo -n "")
 
       - name: "Output generated matrix"
         uses: sergeysova/jq-action@v2

laminas-ci.schema.json

@@ -183,6 +183,9 @@
                 "stablePHP": {
                     "$ref": "#/definitions/stablePHP"
                 },
+                "backwardCompatibilityCheck": {
+                    "$ref": "#/definitions/backwardCompatibilityCheck"
+                },
                 "additional_checks": {
                     "type": "array",
                     "title": "A list of additional checks to be executed",
@@ -328,14 +331,20 @@
             }
         },
         "stablePHP": {
-                "type": "string",
-                "minLength": 1,
-                "title": "The PHP version to be used for stable checks",
-                "description": "This PHP version is used for all QA check jobs. The default depends on the `composer.json` of the project and usually reflects the minimum supported PHP version of that project.",
-                "examples": [
-                    "8.0"
-                ]
-            },
+            "type": "string",
+            "minLength": 1,
+            "title": "The PHP version to be used for stable checks",
+            "description": "This PHP version is used for all QA check jobs. The default depends on the `composer.json` of the project and usually reflects the minimum supported PHP version of that project.",
+            "examples": [
+                "8.0"
+            ]
+        },
+        "backwardCompatibilityCheck": {
+            "type": "boolean",
+            "title": "Flag to enable/disable backwards compatibility check",
+            "description": "This flag enables/disables backwards compatibility check using roave/backward-compatibility-check.",
+            "default": false
+        },
         "job": {
             "type": "object",
             "title": "The job to be executed",

package.json

@@ -23,6 +23,7 @@
         "@types/webpack": "^5.28.0",
         "@typescript-eslint/eslint-plugin": "^5.41.0",
         "@typescript-eslint/parser": "^5.41.0",
+        "dotenv": "^16.3.1",
         "eslint": "^8.26.0",
         "eslint-config-incredible": "^2.4.2",
         "eslint-import-resolver-typescript": "^3.5.2",

src/config/app.spec.ts

@@ -1,6 +1,14 @@
 import {PathLike} from 'fs';
+import {configDotenv} from 'dotenv';
 import createConfig, {gatherVersions} from './app';
 
+beforeEach(() => {
+    jest.resetModules();
+
+    // Clean enviroment to avoid side-effects
+    process.env = {};
+});
+
 describe('config/app', () => {
     describe('gatherVersions()', () => {
         test.each`
@@ -25,6 +33,7 @@ describe('config/app', () => {
 
     describe('createConfig()', () => {
         const phpIniFromConfigurationPath: PathLike = 'tests/php-ini-from-configuration';
+        const roaveBackwardCompatibilityPath: PathLike = 'tests/code-check-roave-backward-compatibility';
 
         it('should return valid config', () => {
             expect(createConfig(
@@ -47,7 +56,41 @@ describe('config/app', () => {
                 phpIni                        : [ 'error_reporting=E_ALL' ],
                 ignorePhpPlatformRequirements : {},
                 additionalComposerArguments   : [],
+                backwardCompatibilityCheck    : false,
+                baseReference                 : null,
+            });
+        });
+
+        it('should detect GITHUB_BASE_REF', () => {
+            const environment = process.env;
+
+            configDotenv({path: `${roaveBackwardCompatibilityPath}/test.env`});
+
+            expect(createConfig(
+                {
+                    codeChecks : true,
+                    docLinting : true,
+                },
+                `${roaveBackwardCompatibilityPath}/composer.json`,
+                `${roaveBackwardCompatibilityPath}/composer.lock`,
+                `${roaveBackwardCompatibilityPath}/.laminas-ci.json`
+            )).toEqual({
+                codeChecks                    : true,
+                docLinting                    : true,
+                versions                      : [],
+                stablePhpVersion              : '7.4',
+                minimumPhpVersion             : '7.4',
+                latestPhpVersion              : '7.4',
+                lockedDependenciesExists      : false,
+                phpExtensions                 : [],
+                phpIni                        : [],
+                ignorePhpPlatformRequirements : {},
+                additionalComposerArguments   : [],
+                backwardCompatibilityCheck    : true,
+                baseReference                 : '1111222233334444aaaabbbbccccdddd',
             });
+
+            process.env = environment;
         });
     });
 });

src/config/app.ts

@@ -1,7 +1,7 @@
 import fs, {PathLike} from 'fs';
 import semver from 'semver';
 import parseJsonFile from '../json';
-import {Tool, ToolExecutionType} from '../tools';
+import {isToolRunningContainerDefaultPhpVersion, Tool, ToolExecutionType} from '../tools';
 import {Logger} from '../logging';
 import {CURRENT_STABLE, INSTALLABLE_VERSIONS, InstallablePhpVersionType, isInstallableVersion} from './php';
 import {ComposerJson} from './composer';
@@ -85,6 +85,8 @@ export interface Config {
     readonly phpIni: string[];
     readonly ignorePhpPlatformRequirements: IgnorePhpPlatformRequirements;
     readonly additionalComposerArguments: string[];
+    readonly backwardCompatibilityCheck: boolean;
+    readonly baseReference: string|null;
 }
 export interface Requirements {
     readonly codeChecks: boolean;
@@ -283,6 +285,17 @@ function createJob(
     return createdJob;
 }
 
+function detectPhpVersionForTool(
+    tool: Tool,
+    config: Config
+): InstallablePhpVersionType {
+    if (isToolRunningContainerDefaultPhpVersion(tool)) {
+        return tool.php;
+    }
+
+    return config.minimumPhpVersion;
+}
+
 function createJobsForTool(
     config: Config,
     tool: Tool
@@ -302,7 +315,7 @@ function createJobsForTool(
                 tool.name,
                 createJobDefinition(
                     tool.command,
-                    config.minimumPhpVersion,
+                    detectPhpVersionForTool(tool, config),
                     lockedOrLatestDependencySet,
                     config.phpExtensions,
                     config.phpIni,
@@ -466,6 +479,8 @@ export default function createConfig(
         lockedDependenciesExists      : fs.existsSync(composerLockJsonFileName),
         ignorePhpPlatformRequirements : configurationFromFile.ignore_php_platform_requirements ?? {},
         additionalComposerArguments   : [ ... new Set(configurationFromFile.additional_composer_arguments ?? []) ],
+        backwardCompatibilityCheck    : configurationFromFile.backwardCompatibilityCheck ?? false,
+        baseReference                 : process.env.GITHUB_BASE_REF ?? null,
     };
 }
 

src/config/input.ts

@@ -11,6 +11,7 @@ export interface ConfigurationFromFile {
     ignore_php_platform_requirements?: IgnorePhpPlatformRequirements;
     stablePHP?: string;
     additional_composer_arguments?: string[];
+    backwardCompatibilityCheck?: boolean;
 }
 
 export interface JobExclusionsFromFile {

src/config/php.ts

@@ -9,6 +9,7 @@ export const PHP_81 = '8.1';
 export const PHP_82 = '8.2';
 
 export const CURRENT_STABLE = PHP_80;
+export const CONTAINER_DEFAULT_PHP_VERSION = '@default';
 
 /**
  * NOTE: Please keep this list ordered as the ordering is used to detect the minimum supported version of a project
@@ -23,7 +24,8 @@ export const INSTALLABLE_VERSIONS = [
     PHP_74,
     PHP_80,
     PHP_81,
-    PHP_82
+    PHP_82,
+    CONTAINER_DEFAULT_PHP_VERSION,
 ] as const;
 
 export type InstallablePhpVersionType = typeof INSTALLABLE_VERSIONS[number];

src/tools.ts

@@ -1,7 +1,8 @@
-import fs, { PathLike } from 'fs';
-import { Config } from './config/app';
-import { ComposerJson } from './config/composer';
+import fs, {PathLike} from 'fs';
+import {Config} from './config/app';
+import {ComposerJson} from './config/composer';
 import parseJsonFile from './json';
+import {CONTAINER_DEFAULT_PHP_VERSION} from './config/php';
 
 export enum ToolExecutionType {
     /**
@@ -31,6 +32,10 @@ export type Tool = {
     lintConfigCommand?: string,
 }
 
+export type ToolRunningContainerDefaultPhpVersion = Tool & {
+    php: typeof CONTAINER_DEFAULT_PHP_VERSION,
+}
+
 function detectInfectionCommand(): string {
     const composerJson: ComposerJson = parseJsonFile('composer.json', true) as ComposerJson;
 
@@ -41,8 +46,28 @@ function detectInfectionCommand(): string {
     return 'phpdbg -qrr ./vendor/bin/infection';
 }
 
+function backwardCompatibilityCheckTool(config: Config): ToolRunningContainerDefaultPhpVersion | null {
+    if (!config.backwardCompatibilityCheck) {
+        return null;
+    }
+
+    if (config.baseReference === null) {
+        return null;
+    }
+
+    return {
+        // @TODO need to `git fetch baseSha1` from source repo!
+        executionType : ToolExecutionType.STATIC,
+        name          : 'Backward Compatibility Check',
+        command       : `roave-backward-compatibility-check check --from="${ config.baseReference }" --install-development-dependencies`,
+        filesToCheck  : [ 'composer.json' ],
+        toolType      : ToolType.CODE_CHECK,
+        php           : CONTAINER_DEFAULT_PHP_VERSION,
+    } as ToolRunningContainerDefaultPhpVersion;
+}
+
 export default function createTools(config: Config): Array<Tool> {
-    return [
+    const tools = [
         {
             executionType : ToolExecutionType.STATIC,
             name          : 'Documentation Linting',
@@ -129,8 +154,11 @@ export default function createTools(config: Config): Array<Tool> {
             command       : './vendor/bin/php-cs-fixer fix -v --diff --dry-run',
             filesToCheck  : [ '.php-cs-fixer.php', '.php-cs-fixer.dist.php' ],
             toolType      : ToolType.CODE_CHECK,
-        }
-    ]
+        },
+        backwardCompatibilityCheckTool(config),
+    ].filter((tool) => tool !== null) as Tool[];
+
+    return tools
         // Remove all tools which do not need to run
         .filter((tool) =>
             (config.docLinting && tool.toolType === ToolType.LINTER)
@@ -146,3 +174,7 @@ export function removeNonExistentFilesToCheck(tool: Tool): Tool {
         filesToCheck : tool.filesToCheck.filter((file) => fs.existsSync(file))
     };
 }
+
+export function isToolRunningContainerDefaultPhpVersion(tool: Tool): tool is ToolRunningContainerDefaultPhpVersion {
+    return (tool as ToolRunningContainerDefaultPhpVersion).php === CONTAINER_DEFAULT_PHP_VERSION;
+}

tests/code-check-roave-backward-compatibility/.laminas-ci.json

@@ -0,0 +1,4 @@
+{
+    "stablePHP": "7.4",
+    "backwardCompatibilityCheck": true
+}

tests/code-check-roave-backward-compatibility/composer.json

@@ -0,0 +1 @@
+{}

tests/code-check-roave-backward-compatibility/matrix.json

@@ -0,0 +1,10 @@
+{
+    "include": [
+        {
+            "name": "Backward Compatibility Check [@default, latest]",
+            "job": "{\"command\":\"roave-backward-compatibility-check check --from=\\\"1111222233334444aaaabbbbccccdddd\\\" --install-development-dependencies\",\"php\":\"@default\",\"extensions\":[],\"ini\":[],\"dependencies\":\"latest\",\"ignore_platform_reqs_8\":false,\"ignore_php_platform_requirement\":false,\"additional_composer_arguments\":[],\"before_script\":[]}",
+            "operatingSystem": "ubuntu-latest",
+            "action": "laminas/laminas-continuous-integration-action@v1"
+        }
+    ]
+}

tests/code-check-roave-backward-compatibility/test.env

@@ -0,0 +1 @@
+GITHUB_BASE_REF=1111222233334444aaaabbbbccccdddd