From 46aac867e2bc9f12c53eae1897e79c051831d015 Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Mon, 17 Jul 2023 16:28:48 +0100
Subject: [PATCH 1/9] Better errors

---
 src/utils/constants.ts      |  2 ++
 src/utils/workflow.utils.ts | 41 +++++++++++++++++++++++++++++++------
 2 files changed, 37 insertions(+), 6 deletions(-)

diff --git a/src/utils/constants.ts b/src/utils/constants.ts
index a6b5fd01..71d2b444 100644
--- a/src/utils/constants.ts
+++ b/src/utils/constants.ts
@@ -14,3 +14,5 @@ export const EXPECTED_PERMISSIONS_BLOCK = [
 // the environment changes in a way that would cause a replay to behave differently, e.g. upgrading to a newer
 // replay-orchestrator-launcher version, or changing the version of puppeteer.
 export const LOGICAL_ENVIRONMENT_VERSION = 2;
+
+export const DOCS_URL = "https://app.meticulous.ai/docs/github-actions-v2";
diff --git a/src/utils/workflow.utils.ts b/src/utils/workflow.utils.ts
index 0ac632a7..91690088 100644
--- a/src/utils/workflow.utils.ts
+++ b/src/utils/workflow.utils.ts
@@ -3,6 +3,7 @@ import { GitHub } from "@actions/github/lib/utils";
 import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
 import log from "loglevel";
 import { DateTime, Duration } from "luxon";
+import { DOCS_URL } from "./constants";
 
 // The GitHub REST API will not list a workflow run immediately after it has been dispatched
 const LISTING_AFTER_DISPATCH_DELAY = Duration.fromObject({ seconds: 10 });
@@ -42,12 +43,40 @@ export const startNewWorkflowRun = async ({
   commitSha: string;
   octokit: InstanceType<typeof GitHub>;
 }): Promise<{ workflowRunId: number; [key: string]: unknown } | undefined> => {
-  await octokit.rest.actions.createWorkflowDispatch({
-    owner,
-    repo,
-    workflow_id: workflowId,
-    ref,
-  });
+  try {
+    await octokit.rest.actions.createWorkflowDispatch({
+      owner,
+      repo,
+      workflow_id: workflowId,
+      ref,
+    });
+  } catch (err: unknown) {
+    const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+    if (
+      (err as { message?: string } | null)?.message?.includes(
+        "Workflow does not have 'workflow_dispatch' trigger"
+      )
+    ) {
+      logger.error(
+        `Could not trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against, because there was no Meticulous workflow with the 'workflow_dispatch' trigger on the ${ref} branch.` +
+          ` Screenshots of the new flows will be taken, but no comparisons will be made.` +
+          ` If you haven't merged the PR to setup Meticulous in Github Actions to the ${ref} branch yet then this is expected.` +
+          ` Otherwise please check that Meticulous is running on the ${ref} branch, that it has a 'workflow_dispatch' trigger, and has the appropiate permissions.` +
+          ` See ${DOCS_URL} for the correct setup.`
+      );
+      logger.debug(err);
+      return undefined;
+    }
+    logger.error(
+      `Could not trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against.` +
+        ` Screenshots of the new flows will be taken, but no comparisons will be made.` +
+        ` Please check that Meticulous is running on the ${ref} branch, that it has a 'workflow_dispatch' trigger, and has the appropiate permissions.` +
+        ` See ${DOCS_URL} for the correct setup.`,
+      err
+    );
+    return undefined;
+  }
+
   // Wait before listing again
   await delay(LISTING_AFTER_DISPATCH_DELAY);
 

From d488e6ecbbe4b7ccbd89582db8257fb204070293 Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Mon, 17 Jul 2023 19:48:47 +0100
Subject: [PATCH 2/9] Fix

---
 src/utils/workflow.utils.ts | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/utils/workflow.utils.ts b/src/utils/workflow.utils.ts
index 91690088..69c9bfd8 100644
--- a/src/utils/workflow.utils.ts
+++ b/src/utils/workflow.utils.ts
@@ -52,10 +52,9 @@ export const startNewWorkflowRun = async ({
     });
   } catch (err: unknown) {
     const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+    const message = (err as { message?: string } | null)?.message ?? "";
     if (
-      (err as { message?: string } | null)?.message?.includes(
-        "Workflow does not have 'workflow_dispatch' trigger"
-      )
+      message.includes("Workflow does not have 'workflow_dispatch' trigger")
     ) {
       logger.error(
         `Could not trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against, because there was no Meticulous workflow with the 'workflow_dispatch' trigger on the ${ref} branch.` +
@@ -67,6 +66,16 @@ export const startNewWorkflowRun = async ({
       logger.debug(err);
       return undefined;
     }
+    if (message.includes("Resource not accessible by integration")) {
+      logger.error(
+        `Does not have permission to trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against, because the 'actions: write' permission is missing in your workflow YAML file.` +
+          ` Screenshots of the new flows will be taken, but no comparisons will be made.` +
+          ` Please check that the Meticulous workflow has the correct permissions: see ${DOCS_URL} for the correct setup.`
+      );
+      logger.debug(err);
+      return undefined;
+    }
+
     logger.error(
       `Could not trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against.` +
         ` Screenshots of the new flows will be taken, but no comparisons will be made.` +

From 7ccce6224cda438959e9c31c72f8b3b05a26f9dc Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Mon, 17 Jul 2023 19:49:47 +0100
Subject: [PATCH 3/9] Add actions write permission

---
 .github/workflows/test-meticulous-against-deployment.yaml | 1 +
 .github/workflows/test-meticulous.yaml                    | 1 +
 2 files changed, 2 insertions(+)

diff --git a/.github/workflows/test-meticulous-against-deployment.yaml b/.github/workflows/test-meticulous-against-deployment.yaml
index c677afb0..bf2060af 100644
--- a/.github/workflows/test-meticulous-against-deployment.yaml
+++ b/.github/workflows/test-meticulous-against-deployment.yaml
@@ -9,6 +9,7 @@ on:
   pull_request: {}
 
 permissions:
+  actions: write
   contents: read
   statuses: write
   pull-requests: write
diff --git a/.github/workflows/test-meticulous.yaml b/.github/workflows/test-meticulous.yaml
index 5249ba88..411b707d 100644
--- a/.github/workflows/test-meticulous.yaml
+++ b/.github/workflows/test-meticulous.yaml
@@ -9,6 +9,7 @@ on:
   pull_request: {}
 
 permissions:
+  actions: write
   contents: read
   statuses: write
   pull-requests: write

From 52f870590ef685e6f7a6afab5c20289e2bddbad2 Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Mon, 17 Jul 2023 19:58:17 +0100
Subject: [PATCH 4/9] Improvements

---
 src/action.ts               |  4 +---
 src/utils/logger.utils.ts   |  2 ++
 src/utils/workflow.utils.ts | 13 +++++++++----
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/src/action.ts b/src/action.ts
index 8bd467ac..af87359b 100644
--- a/src/action.ts
+++ b/src/action.ts
@@ -17,7 +17,7 @@ import { getEnvironment } from "./utils/environment.utils";
 import { getBaseAndHeadCommitShas } from "./utils/get-base-and-head-commit-shas";
 import { getCodeChangeEvent } from "./utils/get-code-change-event";
 import { getInputs } from "./utils/get-inputs";
-import { initLogger, setLogLevel } from "./utils/logger.utils";
+import { initLogger, setLogLevel, shortSha } from "./utils/logger.utils";
 import { ResultsReporter } from "./utils/results-reporter";
 import { waitForDeploymentUrl } from "./utils/wait-for-deployment-url";
 
@@ -224,5 +224,3 @@ const getOctokitOrFail = (githubToken: string | null) => {
     );
   }
 };
-
-const shortSha = (sha: string) => sha.slice(0, 7);
diff --git a/src/utils/logger.utils.ts b/src/utils/logger.utils.ts
index c02b57ec..c1b76d28 100644
--- a/src/utils/logger.utils.ts
+++ b/src/utils/logger.utils.ts
@@ -33,3 +33,5 @@ export const setLogLevel: (logLevel: string | undefined) => void = (
       break;
   }
 };
+
+export const shortSha = (sha: string) => sha.slice(0, 7);
diff --git a/src/utils/workflow.utils.ts b/src/utils/workflow.utils.ts
index 69c9bfd8..3925ab0a 100644
--- a/src/utils/workflow.utils.ts
+++ b/src/utils/workflow.utils.ts
@@ -4,6 +4,7 @@ import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
 import log from "loglevel";
 import { DateTime, Duration } from "luxon";
 import { DOCS_URL } from "./constants";
+import { shortSha } from "./logger.utils";
 
 // The GitHub REST API will not list a workflow run immediately after it has been dispatched
 const LISTING_AFTER_DISPATCH_DELAY = Duration.fromObject({ seconds: 10 });
@@ -57,7 +58,9 @@ export const startNewWorkflowRun = async ({
       message.includes("Workflow does not have 'workflow_dispatch' trigger")
     ) {
       logger.error(
-        `Could not trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against, because there was no Meticulous workflow with the 'workflow_dispatch' trigger on the ${ref} branch.` +
+        `Could not trigger a workflow run on commit ${shortSha(
+          commitSha
+        )} of the base branch (${ref}) to compare against, because there was no Meticulous workflow with the 'workflow_dispatch' trigger on the ${ref} branch.` +
           ` Screenshots of the new flows will be taken, but no comparisons will be made.` +
           ` If you haven't merged the PR to setup Meticulous in Github Actions to the ${ref} branch yet then this is expected.` +
           ` Otherwise please check that Meticulous is running on the ${ref} branch, that it has a 'workflow_dispatch' trigger, and has the appropiate permissions.` +
@@ -68,16 +71,18 @@ export const startNewWorkflowRun = async ({
     }
     if (message.includes("Resource not accessible by integration")) {
       logger.error(
-        `Does not have permission to trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against, because the 'actions: write' permission is missing in your workflow YAML file.` +
+        `Missing permission to trigger a workflow run on the base branch (${ref}).` +
           ` Screenshots of the new flows will be taken, but no comparisons will be made.` +
-          ` Please check that the Meticulous workflow has the correct permissions: see ${DOCS_URL} for the correct setup.`
+          ` Please add the 'actions: write' permission to your workflow YAML file: see ${DOCS_URL} for the correct setup.`
       );
       logger.debug(err);
       return undefined;
     }
 
     logger.error(
-      `Could not trigger a workflow run on ${commitSha} of the base branch (${ref}) to compare against.` +
+      `Could not trigger a workflow run on commit ${shortSha(
+        commitSha
+      )} of the base branch (${ref}) to compare against.` +
         ` Screenshots of the new flows will be taken, but no comparisons will be made.` +
         ` Please check that Meticulous is running on the ${ref} branch, that it has a 'workflow_dispatch' trigger, and has the appropiate permissions.` +
         ` See ${DOCS_URL} for the correct setup.`,

From a487d6c75477f2501d6ec7c8bd28fa70d64435e9 Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Tue, 18 Jul 2023 11:46:24 +0100
Subject: [PATCH 5/9] Improvements to more errors

---
 src/utils/ensure-base-exists.utils.ts | 32 +++++++++---
 src/utils/error.utils.ts              | 25 ++++++++++
 src/utils/results-reporter.ts         | 46 ++++++++++++-----
 src/utils/update-status-comment.ts    | 71 +++++++++++++++++----------
 src/utils/wait-for-deployment-url.ts  | 15 ++++--
 src/utils/workflow.utils.ts           |  4 +-
 6 files changed, 143 insertions(+), 50 deletions(-)
 create mode 100644 src/utils/error.utils.ts

diff --git a/src/utils/ensure-base-exists.utils.ts b/src/utils/ensure-base-exists.utils.ts
index b324fffd..f2d8448d 100644
--- a/src/utils/ensure-base-exists.utils.ts
+++ b/src/utils/ensure-base-exists.utils.ts
@@ -9,7 +9,8 @@ import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
 import log from "loglevel";
 import { Duration } from "luxon";
 import { CodeChangeEvent } from "../types";
-import { LOGICAL_ENVIRONMENT_VERSION } from "./constants";
+import { DOCS_URL, LOGICAL_ENVIRONMENT_VERSION } from "./constants";
+import { isGithubPermissionsError } from "./error.utils";
 import {
   getCurrentWorkflowId,
   getPendingWorkflowRun,
@@ -251,11 +252,26 @@ const getHeadCommitForRef = async ({
   ref: string;
   octokit: InstanceType<typeof GitHub>;
 }): Promise<string> => {
-  const result = await octokit.rest.repos.getBranch({
-    owner,
-    repo,
-    branch: ref,
-  });
-  const commitSha = result.data.commit.sha;
-  return commitSha;
+  try {
+    const result = await octokit.rest.repos.getBranch({
+      owner,
+      repo,
+      branch: ref,
+    });
+    const commitSha = result.data.commit.sha;
+    return commitSha;
+  } catch (err: unknown) {
+    if (isGithubPermissionsError(err)) {
+      // https://docs.github.com/en/rest/overview/permissions-required-for-github-apps?apiVersion=2022-11-28#repository-permissions-for-contents
+      throw new Error(
+        `Missing permission to get the head commit of the branch '${ref}'. This is required in order to correctly calculate the two commits to compare.` +
+          ` Please add the 'contents: read' permission to your workflow YAML file: see ${DOCS_URL} for the correct setup.`
+      );
+    }
+    const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+    logger.error(
+      `Unable to get head commit of branch '${ref}'. This is required in order to correctly calculate the two commits to compare. Please check that you have setup the correct permissions: see ${DOCS_URL} for the correct setup.`
+    );
+    throw err;
+  }
 };
diff --git a/src/utils/error.utils.ts b/src/utils/error.utils.ts
new file mode 100644
index 00000000..3682d58c
--- /dev/null
+++ b/src/utils/error.utils.ts
@@ -0,0 +1,25 @@
+export const isGithubPermissionsError = (
+  error: MaybeOctokitRequestError | unknown
+): boolean => {
+  const message = getErrorMessage(error);
+  return (
+    message.toLowerCase().includes("resource not accessible by integration") ||
+    (error as MaybeOctokitRequestError)?.status === 403
+  );
+};
+
+export const getErrorMessage = (error: unknown) => {
+  const message = (error as MaybeOctokitRequestError)?.message ?? "";
+  return typeof message === "string" ? message : "";
+};
+
+// Octokit returns RequestErrors, which have a message and a code
+// But we can't make any strong assumptions, so we type it on the safe side
+// https://github.com/octokit/request-error.js/blob/372097e9b16f70d4ad75089003dc9154e304faa7/src/index.ts#L16
+type MaybeOctokitRequestError =
+  | {
+      message?: string | unknown;
+      status?: number | unknown;
+    }
+  | null
+  | undefined;
diff --git a/src/utils/results-reporter.ts b/src/utils/results-reporter.ts
index d68edf2c..b9beae6f 100644
--- a/src/utils/results-reporter.ts
+++ b/src/utils/results-reporter.ts
@@ -1,10 +1,15 @@
 import { getOctokit } from "@actions/github";
 import { Project } from "@alwaysmeticulous/api";
+import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
 import {
   ExecuteTestRunResult,
   RunningTestRunExecution,
 } from "@alwaysmeticulous/sdk-bundles-api";
+import log from "loglevel";
 import { CodeChangeEvent } from "../types";
+import { DOCS_URL } from "./constants";
+import { isGithubPermissionsError } from "./error.utils";
+import { shortSha } from "./logger.utils";
 import { updateStatusComment } from "./update-status-comment";
 
 const SHORT_SHA_LENGTH = 7;
@@ -160,18 +165,35 @@ export class ResultsReporter {
     targetUrl?: string;
   }) {
     const { octokit, owner, repo, headSha } = this.options;
-    return octokit.rest.repos.createCommitStatus({
-      owner,
-      repo,
-      context:
-        this.options.testSuiteId != null
-          ? `Meticulous (${this.options.testSuiteId})`
-          : "Meticulous",
-      description,
-      state,
-      sha: headSha,
-      ...(targetUrl ? { target_url: targetUrl } : {}),
-    });
+    try {
+      return octokit.rest.repos.createCommitStatus({
+        owner,
+        repo,
+        context:
+          this.options.testSuiteId != null
+            ? `Meticulous (${this.options.testSuiteId})`
+            : "Meticulous",
+        description,
+        state,
+        sha: headSha,
+        ...(targetUrl ? { target_url: targetUrl } : {}),
+      });
+    } catch (err: unknown) {
+      if (isGithubPermissionsError(err)) {
+        // https://docs.github.com/en/actions/using-jobs/assigning-permissions-to-jobs
+        throw new Error(
+          `Missing permission to create and update commit statuses.` +
+            ` Please add the 'statuses: write' permission to your workflow YAML file: see ${DOCS_URL} for the correct setup.`
+        );
+      }
+      const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+      logger.error(
+        `Unable to create commit status for commit '${shortSha(
+          headSha
+        )}'. Please check that you have setup the correct permissions: see ${DOCS_URL} for the correct setup.`
+      );
+      throw err;
+    }
   }
 
   private setStatusComment({
diff --git a/src/utils/update-status-comment.ts b/src/utils/update-status-comment.ts
index 4b88f7e8..93f9c6d7 100644
--- a/src/utils/update-status-comment.ts
+++ b/src/utils/update-status-comment.ts
@@ -1,5 +1,9 @@
 import { getOctokit } from "@actions/github";
+import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
+import log from "loglevel";
 import { CodeChangeEvent } from "../types";
+import { DOCS_URL } from "./constants";
+import { isGithubPermissionsError } from "./error.utils";
 
 const getMeticulousCommentIdentifier = (testSuiteId: string | null) =>
   `<!--- alwaysmeticulous/report-diffs-action/status-comment${
@@ -30,35 +34,50 @@ export const updateStatusComment = async ({
   }
 
   // Check for existing comments
-  const comments = await octokit.rest.issues.listComments({
-    owner,
-    repo,
-    issue_number: event.payload.pull_request.number,
-    per_page: 1000,
-  });
-  const commentIdentifier = getMeticulousCommentIdentifier(testSuiteId);
-  const existingComment = comments.data.find(
-    (comment) => (comment.body ?? "").indexOf(commentIdentifier) > -1
-  );
-  const testSuiteDescription = testSuiteId
-    ? `Test suite: ${testSuiteId}. `
-    : "";
-
-  const fullBody = `${body}\n\n<sub>${testSuiteDescription}Last updated for commit ${shortHeadSha}. This comment will update as new commits are pushed.</sub>${commentIdentifier}`;
-
-  if (existingComment != null) {
-    await octokit.rest.issues.updateComment({
-      owner,
-      repo,
-      comment_id: existingComment.id,
-      body: fullBody,
-    });
-  } else if (createIfDoesNotExist) {
-    await octokit.rest.issues.createComment({
+  try {
+    const comments = await octokit.rest.issues.listComments({
       owner,
       repo,
       issue_number: event.payload.pull_request.number,
-      body: fullBody,
+      per_page: 1000,
     });
+
+    const commentIdentifier = getMeticulousCommentIdentifier(testSuiteId);
+    const existingComment = comments.data.find(
+      (comment) => (comment.body ?? "").indexOf(commentIdentifier) > -1
+    );
+    const testSuiteDescription = testSuiteId
+      ? `Test suite: ${testSuiteId}. `
+      : "";
+
+    const fullBody = `${body}\n\n<sub>${testSuiteDescription}Last updated for commit ${shortHeadSha}. This comment will update as new commits are pushed.</sub>${commentIdentifier}`;
+
+    if (existingComment != null) {
+      await octokit.rest.issues.updateComment({
+        owner,
+        repo,
+        comment_id: existingComment.id,
+        body: fullBody,
+      });
+    } else if (createIfDoesNotExist) {
+      await octokit.rest.issues.createComment({
+        owner,
+        repo,
+        issue_number: event.payload.pull_request.number,
+        body: fullBody,
+      });
+    }
+  } catch (err) {
+    if (isGithubPermissionsError(err)) {
+      // https://docs.github.com/en/rest/overview/permissions-required-for-github-apps?apiVersion=2022-11-28#repository-permissions-for-pull-requests
+      throw new Error(
+        `Missing permission to list and post comments to the pull request #${event.payload.pull_request.number}. Please add the 'pull-requests: write' permission to your workflow YAML file: see ${DOCS_URL} for the correct setup.`
+      );
+    }
+    const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+    logger.error(
+      `Unable to post / update comment on PR #${event.payload.pull_request.number}. Please check that you have setup the correct permissions: see ${DOCS_URL} for the correct setup.`
+    );
+    throw err;
   }
 };
diff --git a/src/utils/wait-for-deployment-url.ts b/src/utils/wait-for-deployment-url.ts
index ef8e8e39..9e34d012 100644
--- a/src/utils/wait-for-deployment-url.ts
+++ b/src/utils/wait-for-deployment-url.ts
@@ -1,8 +1,11 @@
 import { getOctokit } from "@actions/github";
 import { GitHub } from "@actions/github/lib/utils";
+import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
 import { Hub } from "@sentry/node";
 import { Transaction } from "@sentry/types";
-import { EXPECTED_PERMISSIONS_BLOCK } from "./constants";
+import log from "loglevel";
+import { DOCS_URL, EXPECTED_PERMISSIONS_BLOCK } from "./constants";
+import { isGithubPermissionsError } from "./error.utils";
 
 const TIMEOUT_MS = 30 * 60 * 1_000; // 30 minutes
 const MIN_POLL_FREQUENCY = 1_000;
@@ -101,15 +104,21 @@ const getDeploymentUrl = async ({
       per_page: MAX_GITHUB_ALLOWED_PAGE_SIZE,
     });
   } catch (err) {
-    console.error("Error listing deployments", err);
+    if (!isGithubPermissionsError(err)) {
+      // If it's a permission error our main error message is sufficient, we don't need to log the raw github one,
+      // but otherwise we should log it:
+      console.error("Error listing deployments", err);
+    }
   }
 
   if (deployments == null || deployments.status !== 200) {
+    // https://docs.github.com/en/rest/overview/permissions-required-for-github-apps?apiVersion=2022-11-28#repository-permissions-for-deployments
     throw new Error(
       `Failed to list deployments for commit ${commitSha}.\n\n` +
         "Note: if using 'use-deployment-url' then you must provide permissions for the action to read deployments. " +
         "To do this edit the 'permissions:' block in your workflow file to include 'deployments: read'. Your permissions block should look like:\n\n" +
-        EXPECTED_PERMISSIONS_BLOCK
+        EXPECTED_PERMISSIONS_BLOCK +
+        "\n\nSee ${DOCS_URL} for the correct setup."
     );
   }
 
diff --git a/src/utils/workflow.utils.ts b/src/utils/workflow.utils.ts
index 3925ab0a..40548dcd 100644
--- a/src/utils/workflow.utils.ts
+++ b/src/utils/workflow.utils.ts
@@ -4,6 +4,7 @@ import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
 import log from "loglevel";
 import { DateTime, Duration } from "luxon";
 import { DOCS_URL } from "./constants";
+import { isGithubPermissionsError } from "./error.utils";
 import { shortSha } from "./logger.utils";
 
 // The GitHub REST API will not list a workflow run immediately after it has been dispatched
@@ -69,7 +70,8 @@ export const startNewWorkflowRun = async ({
       logger.debug(err);
       return undefined;
     }
-    if (message.includes("Resource not accessible by integration")) {
+    if (isGithubPermissionsError(err)) {
+      // https://docs.github.com/en/rest/overview/permissions-required-for-github-apps?apiVersion=2022-11-28#repository-permissions-for-actions
       logger.error(
         `Missing permission to trigger a workflow run on the base branch (${ref}).` +
           ` Screenshots of the new flows will be taken, but no comparisons will be made.` +

From e2005cd9d227b813c0f5ae6cbf2e617dbb260aa3 Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Tue, 18 Jul 2023 11:50:52 +0100
Subject: [PATCH 6/9] More improvements

---
 src/utils/ensure-base-exists.utils.ts | 7 +++++--
 src/utils/error.utils.ts              | 4 ++++
 src/utils/results-reporter.ts         | 7 +++++--
 src/utils/update-status-comment.ts    | 7 +++++--
 4 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/src/utils/ensure-base-exists.utils.ts b/src/utils/ensure-base-exists.utils.ts
index f2d8448d..960f264f 100644
--- a/src/utils/ensure-base-exists.utils.ts
+++ b/src/utils/ensure-base-exists.utils.ts
@@ -10,7 +10,10 @@ import log from "loglevel";
 import { Duration } from "luxon";
 import { CodeChangeEvent } from "../types";
 import { DOCS_URL, LOGICAL_ENVIRONMENT_VERSION } from "./constants";
-import { isGithubPermissionsError } from "./error.utils";
+import {
+  DEFAULT_FAILED_OCTOKIT_REQUEST_MESSAGE,
+  isGithubPermissionsError,
+} from "./error.utils";
 import {
   getCurrentWorkflowId,
   getPendingWorkflowRun,
@@ -270,7 +273,7 @@ const getHeadCommitForRef = async ({
     }
     const logger = log.getLogger(METICULOUS_LOGGER_NAME);
     logger.error(
-      `Unable to get head commit of branch '${ref}'. This is required in order to correctly calculate the two commits to compare. Please check that you have setup the correct permissions: see ${DOCS_URL} for the correct setup.`
+      `Unable to get head commit of branch '${ref}'. This is required in order to correctly calculate the two commits to compare. ${DEFAULT_FAILED_OCTOKIT_REQUEST_MESSAGE}`
     );
     throw err;
   }
diff --git a/src/utils/error.utils.ts b/src/utils/error.utils.ts
index 3682d58c..f6f8a047 100644
--- a/src/utils/error.utils.ts
+++ b/src/utils/error.utils.ts
@@ -1,3 +1,5 @@
+import { DOCS_URL } from "./constants";
+
 export const isGithubPermissionsError = (
   error: MaybeOctokitRequestError | unknown
 ): boolean => {
@@ -23,3 +25,5 @@ type MaybeOctokitRequestError =
     }
   | null
   | undefined;
+
+export const DEFAULT_FAILED_OCTOKIT_REQUEST_MESSAGE = `Please check www.githubstatus.com, and that you have setup the action correctly, including with the correct permissions: see ${DOCS_URL} for the correct setup.`;
diff --git a/src/utils/results-reporter.ts b/src/utils/results-reporter.ts
index b9beae6f..90e914e7 100644
--- a/src/utils/results-reporter.ts
+++ b/src/utils/results-reporter.ts
@@ -8,7 +8,10 @@ import {
 import log from "loglevel";
 import { CodeChangeEvent } from "../types";
 import { DOCS_URL } from "./constants";
-import { isGithubPermissionsError } from "./error.utils";
+import {
+  DEFAULT_FAILED_OCTOKIT_REQUEST_MESSAGE,
+  isGithubPermissionsError,
+} from "./error.utils";
 import { shortSha } from "./logger.utils";
 import { updateStatusComment } from "./update-status-comment";
 
@@ -190,7 +193,7 @@ export class ResultsReporter {
       logger.error(
         `Unable to create commit status for commit '${shortSha(
           headSha
-        )}'. Please check that you have setup the correct permissions: see ${DOCS_URL} for the correct setup.`
+        )}'. ${DEFAULT_FAILED_OCTOKIT_REQUEST_MESSAGE}`
       );
       throw err;
     }
diff --git a/src/utils/update-status-comment.ts b/src/utils/update-status-comment.ts
index 93f9c6d7..101bcbfe 100644
--- a/src/utils/update-status-comment.ts
+++ b/src/utils/update-status-comment.ts
@@ -3,7 +3,10 @@ import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
 import log from "loglevel";
 import { CodeChangeEvent } from "../types";
 import { DOCS_URL } from "./constants";
-import { isGithubPermissionsError } from "./error.utils";
+import {
+  DEFAULT_FAILED_OCTOKIT_REQUEST_MESSAGE,
+  isGithubPermissionsError,
+} from "./error.utils";
 
 const getMeticulousCommentIdentifier = (testSuiteId: string | null) =>
   `<!--- alwaysmeticulous/report-diffs-action/status-comment${
@@ -76,7 +79,7 @@ export const updateStatusComment = async ({
     }
     const logger = log.getLogger(METICULOUS_LOGGER_NAME);
     logger.error(
-      `Unable to post / update comment on PR #${event.payload.pull_request.number}. Please check that you have setup the correct permissions: see ${DOCS_URL} for the correct setup.`
+      `Unable to post / update comment on PR #${event.payload.pull_request.number}. ${DEFAULT_FAILED_OCTOKIT_REQUEST_MESSAGE}`
     );
     throw err;
   }

From bdfb624916381b78c8adc432067a9fb323b23457 Mon Sep 17 00:00:00 2001
From: Que3216 <q@meticulous.ai>
Date: Wed, 19 Jul 2023 15:06:34 +0100
Subject: [PATCH 7/9] Update src/utils/wait-for-deployment-url.ts

Co-authored-by: Daniel Book <36259973+dbook13@users.noreply.github.com>
---
 src/utils/wait-for-deployment-url.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/utils/wait-for-deployment-url.ts b/src/utils/wait-for-deployment-url.ts
index 9e34d012..648a1ebc 100644
--- a/src/utils/wait-for-deployment-url.ts
+++ b/src/utils/wait-for-deployment-url.ts
@@ -118,7 +118,7 @@ const getDeploymentUrl = async ({
         "Note: if using 'use-deployment-url' then you must provide permissions for the action to read deployments. " +
         "To do this edit the 'permissions:' block in your workflow file to include 'deployments: read'. Your permissions block should look like:\n\n" +
         EXPECTED_PERMISSIONS_BLOCK +
-        "\n\nSee ${DOCS_URL} for the correct setup."
+        `\n\nSee ${DOCS_URL} for the correct setup.`
     );
   }
 

From 5a6d83b9867359c6c56d10f67d9db0effff210bd Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Wed, 19 Jul 2023 15:08:44 +0100
Subject: [PATCH 8/9] Use logger rather than console

---
 src/utils/wait-for-deployment-url.ts | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/src/utils/wait-for-deployment-url.ts b/src/utils/wait-for-deployment-url.ts
index 648a1ebc..84bb3990 100644
--- a/src/utils/wait-for-deployment-url.ts
+++ b/src/utils/wait-for-deployment-url.ts
@@ -28,6 +28,8 @@ export const waitForDeploymentUrl = async ({
   transaction: Transaction;
   allowedEnvironments: string[] | null;
 }): Promise<string> => {
+  const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+
   const waitForDeploymentSpan = transaction.startChild({
     op: "waitForDeployment",
   });
@@ -45,7 +47,7 @@ export const waitForDeploymentUrl = async ({
     });
     deploymentsFound = availableDeployments;
     if (deploymentUrl != null) {
-      console.log(`Testing against deployment URL '${deploymentUrl}'`);
+      logger.log(`Testing against deployment URL '${deploymentUrl}'`);
       return deploymentUrl;
     }
     await new Promise((resolve) => setTimeout(resolve, pollFrequency));
@@ -93,6 +95,8 @@ const getDeploymentUrl = async ({
   deploymentUrl: string | null;
   availableDeployments: DeploymentsArray | null;
 }> => {
+  const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+
   let deployments: Awaited<
     ReturnType<typeof octokit.rest.repos.listDeployments>
   > | null = null;
@@ -107,7 +111,7 @@ const getDeploymentUrl = async ({
     if (!isGithubPermissionsError(err)) {
       // If it's a permission error our main error message is sufficient, we don't need to log the raw github one,
       // but otherwise we should log it:
-      console.error("Error listing deployments", err);
+      logger.error("Error listing deployments", err);
     }
   }
 
@@ -122,7 +126,7 @@ const getDeploymentUrl = async ({
     );
   }
 
-  console.debug(`Found ${deployments.data.length} deployments`);
+  logger.debug(`Found ${deployments.data.length} deployments`);
 
   if (hasMorePages(deployments)) {
     throw new Error(
@@ -154,7 +158,7 @@ const getDeploymentUrl = async ({
 
   if (matchingDeployments.length > 1) {
     if (allowedEnvironments == null) {
-      console.warn(
+      logger.warn(
         `More than one deployment found for commit ${commitSha}: ${describeDeployments(
           matchingDeployments
         )}. Please specify an environment name using the 'allowed-environments' input.`
@@ -174,7 +178,7 @@ const getDeploymentUrl = async ({
     return { deploymentUrl: null, availableDeployments: deployments.data };
   }
 
-  console.debug(`Checking status of deployment ${latestMatchingDeployment.id}`);
+  logger.debug(`Checking status of deployment ${latestMatchingDeployment.id}`);
 
   const deploymentStatuses = await octokit.rest.repos.listDeploymentStatuses({
     owner,

From ce50df669a4f17c941da58261cf6fd0b972aa1ea Mon Sep 17 00:00:00 2001
From: Quentin <q@meticulous.ai>
Date: Wed, 19 Jul 2023 15:11:45 +0100
Subject: [PATCH 9/9] console -> logger

---
 src/action.ts                              | 18 ++++++++++++------
 src/utils/ensure-base-exists.utils.ts      |  6 +++---
 src/utils/get-base-and-head-commit-shas.ts | 19 ++++++++++++-------
 src/utils/wait-for-deployment-url.ts       |  2 +-
 4 files changed, 28 insertions(+), 17 deletions(-)

diff --git a/src/action.ts b/src/action.ts
index af87359b..ba0eb949 100644
--- a/src/action.ts
+++ b/src/action.ts
@@ -1,7 +1,10 @@
 import { setFailed } from "@actions/core";
 import { context, getOctokit } from "@actions/github";
 import { applyDefaultExecutionOptionsFromProject } from "@alwaysmeticulous/client";
-import { setMeticulousLocalDataDir } from "@alwaysmeticulous/common";
+import {
+  METICULOUS_LOGGER_NAME,
+  setMeticulousLocalDataDir,
+} from "@alwaysmeticulous/common";
 import { executeTestRun } from "@alwaysmeticulous/replay-orchestrator-launcher";
 import {
   ReplayExecutionOptions,
@@ -9,6 +12,7 @@ import {
 } from "@alwaysmeticulous/sdk-bundles-api";
 import { initSentry } from "@alwaysmeticulous/sentry";
 import debounce from "lodash.debounce";
+import log from "loglevel";
 import { addLocalhostAliases } from "./utils/add-localhost-aliases";
 import { throwIfCannotConnectToOrigin } from "./utils/check-connection";
 import { LOGICAL_ENVIRONMENT_VERSION } from "./utils/constants";
@@ -73,9 +77,10 @@ export const runMeticulousTestsAction = async (): Promise<void> => {
   const event = getCodeChangeEvent(context.eventName, payload);
   const { owner, repo } = context.repo;
   const octokit = getOctokitOrFail(githubToken);
+  const logger = log.getLogger(METICULOUS_LOGGER_NAME);
 
   if (event == null) {
-    console.warn(
+    logger.warn(
       `Running report-diffs-action is only supported for 'push', \
       'pull_request' and 'workflow_dispatch' events, but was triggered \
       on a '${context.eventName}' event. Skipping execution.`
@@ -97,19 +102,19 @@ export const runMeticulousTestsAction = async (): Promise<void> => {
   });
 
   if (shaToCompareAgainst != null && event.type === "pull_request") {
-    console.log(
+    logger.info(
       `Comparing screenshots for the commit head of this PR, ${shortSha(
         head
       )}, against ${shortSha(shaToCompareAgainst)}`
     );
   } else if (shaToCompareAgainst != null) {
-    console.log(
+    logger.info(
       `Comparing screenshots for commit ${shortSha(
         head
       )} against commit ${shortSha(shaToCompareAgainst)}}`
     );
   } else {
-    console.log(`Generating screenshots for commit ${shortSha(head)}`);
+    logger.info(`Generating screenshots for commit ${shortSha(head)}`);
   }
 
   const resultsReporter = new ResultsReporter({
@@ -218,7 +223,8 @@ const getOctokitOrFail = (githubToken: string | null) => {
   try {
     return getOctokit(githubToken);
   } catch (err) {
-    console.error(err);
+    const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+    logger.error(err);
     throw new Error(
       "Error connecting to GitHub. Did you specify a valid 'github-token'?"
     );
diff --git a/src/utils/ensure-base-exists.utils.ts b/src/utils/ensure-base-exists.utils.ts
index 960f264f..0f7301eb 100644
--- a/src/utils/ensure-base-exists.utils.ts
+++ b/src/utils/ensure-base-exists.utils.ts
@@ -72,7 +72,7 @@ export const ensureBaseTestsExists = async ({
   });
 
   if (testRun != null) {
-    logger.log(`Tests already exist for commit ${base} (${testRun.id})`);
+    logger.info(`Tests already exist for commit ${base} (${testRun.id})`);
     return { shaToCompareAgainst: base };
   }
 
@@ -86,7 +86,7 @@ export const ensureBaseTestsExists = async ({
     octokit,
   });
   if (alreadyPending != null) {
-    logger.log(
+    logger.info(
       `Waiting on workflow run on base commit (${base}) to compare against: ${alreadyPending.html_url}`
     );
 
@@ -167,7 +167,7 @@ export const ensureBaseTestsExists = async ({
     return { shaToCompareAgainst: null };
   }
 
-  logger.log(`Waiting on workflow run: ${workflowRun.html_url}`);
+  logger.info(`Waiting on workflow run: ${workflowRun.html_url}`);
   await waitForWorkflowCompletionAndThrowIfFailed({
     owner,
     repo,
diff --git a/src/utils/get-base-and-head-commit-shas.ts b/src/utils/get-base-and-head-commit-shas.ts
index 0ff4e4b5..d0a4a912 100644
--- a/src/utils/get-base-and-head-commit-shas.ts
+++ b/src/utils/get-base-and-head-commit-shas.ts
@@ -1,5 +1,7 @@
 import { execSync } from "child_process";
 import { context } from "@actions/github";
+import { METICULOUS_LOGGER_NAME } from "@alwaysmeticulous/common";
+import log from "loglevel";
 import { CodeChangeEvent } from "../types";
 
 interface BaseAndHeadCommitShas {
@@ -53,6 +55,8 @@ const tryGetMergeBaseOfHeadCommit = (
   pullRequestBaseSha: string,
   baseRef: string
 ): string | null => {
+  const logger = log.getLogger(METICULOUS_LOGGER_NAME);
+
   try {
     markGitDirectoryAsSafe();
     // Only a single commit is fetched by the checkout action by default
@@ -69,7 +73,7 @@ const tryGetMergeBaseOfHeadCommit = (
     if (!isValidGitSha(mergeBase)) {
       // Note: the GITHUB_SHA is always a merge commit, even if the merge is a no-op because the PR is up to date
       // So this should never happen
-      console.error(
+      logger.error(
         `Failed to get merge base of ${pullRequestHeadSha} and ${baseRef}: value returned by 'git merge-base' was not a valid git SHA ('${mergeBase}').` +
           `Using the base of the pull request instead (${pullRequestBaseSha}).`
       );
@@ -78,7 +82,7 @@ const tryGetMergeBaseOfHeadCommit = (
 
     return mergeBase;
   } catch (error) {
-    console.error(
+    logger.error(
       `Failed to get merge base of ${pullRequestHeadSha} and ${baseRef}. Error: ${error}. Using the base of the pull request instead (${pullRequestBaseSha}).`
     );
     return null;
@@ -90,9 +94,10 @@ const tryGetMergeBaseOfTemporaryMergeCommit = (
   pullRequestBaseSha: string
 ): string | null => {
   const mergeCommitSha = process.env.GITHUB_SHA;
+  const logger = log.getLogger(METICULOUS_LOGGER_NAME);
 
   if (mergeCommitSha == null) {
-    console.error(
+    logger.error(
       `No GITHUB_SHA environment var set, so can't work out true base of the merge commit. Using the base of the pull request instead (${pullRequestBaseSha}).`
     );
     return null;
@@ -104,7 +109,7 @@ const tryGetMergeBaseOfTemporaryMergeCommit = (
       .toString()
       .trim();
     if (headCommitSha !== mergeCommitSha) {
-      console.log(
+      logger.info(
         `The head commit SHA (${headCommitSha}) does not equal GITHUB_SHA environment variable (${mergeCommitSha}).
           This is likely because a custom ref has been passed to the 'actions/checkout' action. We're assuming therefore
           that the head commit SHA is not a temporary merge commit, but rather the head of the branch. Therefore we're
@@ -124,7 +129,7 @@ const tryGetMergeBaseOfTemporaryMergeCommit = (
     if (parents.length !== 2) {
       // Note: the GITHUB_SHA is always a merge commit, even if the merge is a no-op because the PR is up to date
       // So this should never happen
-      console.error(
+      logger.error(
         `GITHUB_SHA (${mergeCommitSha}) is not a merge commit, so can't work out true base of the merge commit. Using the base of the pull request instead.`
       );
       return null;
@@ -134,7 +139,7 @@ const tryGetMergeBaseOfTemporaryMergeCommit = (
     const mergeBaseSha = parents[0];
     const mergeHeadSha = parents[1];
     if (mergeHeadSha !== pullRequestHeadSha) {
-      console.error(
+      logger.error(
         `The second parent (${parents[1]}) of the GITHUB_SHA merge commit (${mergeCommitSha}) is not equal to the head of the PR (${pullRequestHeadSha}),
         so can not confidently determine the base of the merge commit to compare against. Using the base of the pull request instead (${pullRequestBaseSha}).`
       );
@@ -142,7 +147,7 @@ const tryGetMergeBaseOfTemporaryMergeCommit = (
     }
     return mergeBaseSha;
   } catch (e) {
-    console.error(
+    logger.error(
       `Error getting base of merge commit (${mergeCommitSha}). Using the base of the pull request instead (${pullRequestBaseSha}).`,
       e
     );
diff --git a/src/utils/wait-for-deployment-url.ts b/src/utils/wait-for-deployment-url.ts
index 84bb3990..39c95435 100644
--- a/src/utils/wait-for-deployment-url.ts
+++ b/src/utils/wait-for-deployment-url.ts
@@ -47,7 +47,7 @@ export const waitForDeploymentUrl = async ({
     });
     deploymentsFound = availableDeployments;
     if (deploymentUrl != null) {
-      logger.log(`Testing against deployment URL '${deploymentUrl}'`);
+      logger.info(`Testing against deployment URL '${deploymentUrl}'`);
       return deploymentUrl;
     }
     await new Promise((resolve) => setTimeout(resolve, pollFrequency));