From a76773f4b0dfb903141d6ae68d015a326e4eb187 Mon Sep 17 00:00:00 2001 From: Spencer Date: Thu, 3 Sep 2020 10:01:34 -0700 Subject: [PATCH] [release notes] automatically retry on Github API 5xx errors (#76447) Co-authored-by: spalger --- packages/kbn-release-notes/src/cli.ts | 8 +- .../kbn-release-notes/src/lib/classify_pr.ts | 2 +- packages/kbn-release-notes/src/lib/index.ts | 2 +- .../src/lib/irrelevant_pr_summary.ts | 2 +- .../src/lib/is_pr_relevant.ts | 2 +- packages/kbn-release-notes/src/lib/pr_api.ts | 231 ++++++++++++++++++ .../kbn-release-notes/src/lib/pull_request.ts | 206 ---------------- 7 files changed, 239 insertions(+), 214 deletions(-) create mode 100644 packages/kbn-release-notes/src/lib/pr_api.ts delete mode 100644 packages/kbn-release-notes/src/lib/pull_request.ts diff --git a/packages/kbn-release-notes/src/cli.ts b/packages/kbn-release-notes/src/cli.ts index 44b4a7a0282d20..7dcfa380783919 100644 --- a/packages/kbn-release-notes/src/cli.ts +++ b/packages/kbn-release-notes/src/cli.ts @@ -25,8 +25,7 @@ import { run, createFlagError, createFailError, REPO_ROOT } from '@kbn/dev-utils import { FORMATS, SomeFormat } from './formats'; import { - iterRelevantPullRequests, - getPr, + PrApi, Version, ClassifiedPr, streamFromIterable, @@ -48,6 +47,7 @@ export function runReleaseNotesCli() { if (!token || typeof token !== 'string') { throw createFlagError('--token must be defined'); } + const prApi = new PrApi(log, token); const version = Version.fromFlag(flags.version); if (!version) { @@ -80,7 +80,7 @@ export function runReleaseNotesCli() { } const summary = new IrrelevantPrSummary(log); - const pr = await getPr(token, number); + const pr = await prApi.getPr(number); log.success( inspect( { @@ -101,7 +101,7 @@ export function runReleaseNotesCli() { const summary = new IrrelevantPrSummary(log); const prsToReport: ClassifiedPr[] = []; - const prIterable = iterRelevantPullRequests(token, version, log); + const prIterable = prApi.iterRelevantPullRequests(version); for await (const pr of prIterable) { if (!isPrRelevant(pr, version, includeVersions, summary)) { continue; diff --git a/packages/kbn-release-notes/src/lib/classify_pr.ts b/packages/kbn-release-notes/src/lib/classify_pr.ts index c567935ab7e480..2dfe6916235eef 100644 --- a/packages/kbn-release-notes/src/lib/classify_pr.ts +++ b/packages/kbn-release-notes/src/lib/classify_pr.ts @@ -27,7 +27,7 @@ import { ASCIIDOC_SECTIONS, UNKNOWN_ASCIIDOC_SECTION, } from '../release_notes_config'; -import { PullRequest } from './pull_request'; +import { PullRequest } from './pr_api'; export interface ClassifiedPr extends PullRequest { area: Area; diff --git a/packages/kbn-release-notes/src/lib/index.ts b/packages/kbn-release-notes/src/lib/index.ts index 00d8f49cf763fa..8d27a26d96d0a4 100644 --- a/packages/kbn-release-notes/src/lib/index.ts +++ b/packages/kbn-release-notes/src/lib/index.ts @@ -17,7 +17,7 @@ * under the License. */ -export * from './pull_request'; +export * from './pr_api'; export * from './version'; export * from './is_pr_relevant'; export * from './streams'; diff --git a/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts b/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts index 1a458a04c7740d..ba82ab8780465b 100644 --- a/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts +++ b/packages/kbn-release-notes/src/lib/irrelevant_pr_summary.ts @@ -19,7 +19,7 @@ import { ToolingLog } from '@kbn/dev-utils'; -import { PullRequest } from './pull_request'; +import { PullRequest } from './pr_api'; import { Version } from './version'; export class IrrelevantPrSummary { diff --git a/packages/kbn-release-notes/src/lib/is_pr_relevant.ts b/packages/kbn-release-notes/src/lib/is_pr_relevant.ts index af2ef9440dedeb..452a14e919ed44 100644 --- a/packages/kbn-release-notes/src/lib/is_pr_relevant.ts +++ b/packages/kbn-release-notes/src/lib/is_pr_relevant.ts @@ -18,7 +18,7 @@ */ import { Version } from './version'; -import { PullRequest } from './pull_request'; +import { PullRequest } from './pr_api'; import { IGNORE_LABELS } from '../release_notes_config'; import { IrrelevantPrSummary } from './irrelevant_pr_summary'; diff --git a/packages/kbn-release-notes/src/lib/pr_api.ts b/packages/kbn-release-notes/src/lib/pr_api.ts new file mode 100644 index 00000000000000..1f26aa7ad86c36 --- /dev/null +++ b/packages/kbn-release-notes/src/lib/pr_api.ts @@ -0,0 +1,231 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { inspect } from 'util'; + +import Axios from 'axios'; +import gql from 'graphql-tag'; +import * as GraphqlPrinter from 'graphql/language/printer'; +import { DocumentNode } from 'graphql/language/ast'; +import makeTerminalLink from 'terminal-link'; +import { ToolingLog, isAxiosResponseError } from '@kbn/dev-utils'; + +import { Version } from './version'; +import { getFixReferences } from './get_fix_references'; +import { getNoteFromDescription } from './get_note_from_description'; + +const PrNodeFragment = gql` + fragment PrNode on PullRequest { + number + url + title + bodyText + bodyHTML + mergedAt + baseRefName + state + author { + login + ... on User { + name + } + } + labels(first: 100) { + nodes { + name + } + } + } +`; + +export interface PullRequest { + number: number; + url: string; + title: string; + targetBranch: string; + mergedAt: string; + state: string; + labels: string[]; + fixes: string[]; + user: { + name: string; + login: string; + }; + versions: Version[]; + terminalLink: string; + note?: string; +} + +export class PrApi { + constructor(private readonly log: ToolingLog, private readonly token: string) {} + + async getPr(number: number) { + const resp = await this.gqlRequest( + gql` + query($number: Int!) { + repository(owner: "elastic", name: "kibana") { + pullRequest(number: $number) { + ...PrNode + } + } + } + ${PrNodeFragment} + `, + { + number, + } + ); + + const node = resp.data?.repository?.pullRequest; + if (!node) { + throw new Error(`unexpected github response, unable to fetch PR: ${inspect(resp)}`); + } + + return this.parsePullRequestNode(node); + } + + /** + * Iterate all of the PRs which have the `version` label + */ + async *iterRelevantPullRequests(version: Version) { + let nextCursor: string | undefined; + let hasNextPage = true; + + while (hasNextPage) { + const resp = await this.gqlRequest( + gql` + query($cursor: String, $labels: [String!]) { + repository(owner: "elastic", name: "kibana") { + pullRequests(first: 100, after: $cursor, labels: $labels, states: MERGED) { + pageInfo { + hasNextPage + endCursor + } + nodes { + ...PrNode + } + } + } + } + ${PrNodeFragment} + `, + { + cursor: nextCursor, + labels: [version.label], + } + ); + + const pullRequests = resp.data?.repository?.pullRequests; + if (!pullRequests) { + throw new Error(`unexpected github response, unable to fetch PRs: ${inspect(resp)}`); + } + + hasNextPage = pullRequests.pageInfo?.hasNextPage; + nextCursor = pullRequests.pageInfo?.endCursor; + + if (hasNextPage === undefined || (hasNextPage && !nextCursor)) { + throw new Error( + `github response does not include valid pagination information: ${inspect(resp)}` + ); + } + + for (const node of pullRequests.nodes) { + yield this.parsePullRequestNode(node); + } + } + } + + /** + * Convert the Github API response into the structure used by this tool + * + * @param node A GraphQL response from Github using the PrNode fragment + */ + private parsePullRequestNode(node: any): PullRequest { + const terminalLink = makeTerminalLink(`#${node.number}`, node.url); + + const labels: string[] = node.labels.nodes.map((l: { name: string }) => l.name); + + return { + number: node.number, + url: node.url, + terminalLink, + title: node.title, + targetBranch: node.baseRefName, + state: node.state, + mergedAt: node.mergedAt, + labels, + fixes: getFixReferences(node.bodyText), + user: { + login: node.author?.login || 'deleted user', + name: node.author?.name, + }, + versions: labels + .map((l) => Version.fromLabel(l)) + .filter((v): v is Version => v instanceof Version), + note: getNoteFromDescription(node.bodyHTML), + }; + } + + /** + * Send a single request to the Github v4 GraphQL API + */ + private async gqlRequest(query: DocumentNode, variables: Record = {}) { + let attempt = 0; + + while (true) { + attempt += 1; + + try { + const resp = await Axios.request({ + url: 'https://api.github.com/graphql', + method: 'POST', + headers: { + 'user-agent': '@kbn/release-notes', + authorization: `bearer ${this.token}`, + }, + data: { + query: GraphqlPrinter.print(query), + variables, + }, + }); + + return resp.data; + } catch (error) { + if (!isAxiosResponseError(error) || error.response.status < 500) { + // rethrow error unless it is a 500+ response from github + throw error; + } + + const { status, data } = error.response; + const resp = inspect(data); + + if (attempt === 5) { + throw new Error( + `${status} response from Github, attempted request ${attempt} times: [${resp}]` + ); + } + + const delay = attempt * 2000; + this.log.debug(`Github responded with ${status}, retrying in ${delay} ms: [${resp}]`); + await new Promise((resolve) => setTimeout(resolve, delay)); + continue; + } + } + } +} diff --git a/packages/kbn-release-notes/src/lib/pull_request.ts b/packages/kbn-release-notes/src/lib/pull_request.ts deleted file mode 100644 index e61e496642062a..00000000000000 --- a/packages/kbn-release-notes/src/lib/pull_request.ts +++ /dev/null @@ -1,206 +0,0 @@ -/* - * Licensed to Elasticsearch B.V. under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch B.V. licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -import { inspect } from 'util'; - -import Axios from 'axios'; -import gql from 'graphql-tag'; -import * as GraphqlPrinter from 'graphql/language/printer'; -import { DocumentNode } from 'graphql/language/ast'; -import makeTerminalLink from 'terminal-link'; -import { ToolingLog } from '@kbn/dev-utils'; - -import { Version } from './version'; -import { getFixReferences } from './get_fix_references'; -import { getNoteFromDescription } from './get_note_from_description'; - -const PrNodeFragment = gql` - fragment PrNode on PullRequest { - number - url - title - bodyText - bodyHTML - mergedAt - baseRefName - state - author { - login - ... on User { - name - } - } - labels(first: 100) { - nodes { - name - } - } - } -`; - -export interface PullRequest { - number: number; - url: string; - title: string; - targetBranch: string; - mergedAt: string; - state: string; - labels: string[]; - fixes: string[]; - user: { - name: string; - login: string; - }; - versions: Version[]; - terminalLink: string; - note?: string; -} - -/** - * Send a single request to the Github v4 GraphQL API - */ -async function gqlRequest( - token: string, - query: DocumentNode, - variables: Record = {} -) { - const resp = await Axios.request({ - url: 'https://api.github.com/graphql', - method: 'POST', - headers: { - 'user-agent': '@kbn/release-notes', - authorization: `bearer ${token}`, - }, - data: { - query: GraphqlPrinter.print(query), - variables, - }, - }); - - return resp.data; -} - -/** - * Convert the Github API response into the structure used by this tool - * - * @param node A GraphQL response from Github using the PrNode fragment - */ -function parsePullRequestNode(node: any): PullRequest { - const terminalLink = makeTerminalLink(`#${node.number}`, node.url); - - const labels: string[] = node.labels.nodes.map((l: { name: string }) => l.name); - - return { - number: node.number, - url: node.url, - terminalLink, - title: node.title, - targetBranch: node.baseRefName, - state: node.state, - mergedAt: node.mergedAt, - labels, - fixes: getFixReferences(node.bodyText), - user: { - login: node.author?.login || 'deleted user', - name: node.author?.name, - }, - versions: labels - .map((l) => Version.fromLabel(l)) - .filter((v): v is Version => v instanceof Version), - note: getNoteFromDescription(node.bodyHTML), - }; -} - -/** - * Iterate all of the PRs which have the `version` label - */ -export async function* iterRelevantPullRequests(token: string, version: Version, log: ToolingLog) { - let nextCursor: string | undefined; - let hasNextPage = true; - - while (hasNextPage) { - const resp = await gqlRequest( - token, - gql` - query($cursor: String, $labels: [String!]) { - repository(owner: "elastic", name: "kibana") { - pullRequests(first: 100, after: $cursor, labels: $labels, states: MERGED) { - pageInfo { - hasNextPage - endCursor - } - nodes { - ...PrNode - } - } - } - } - ${PrNodeFragment} - `, - { - cursor: nextCursor, - labels: [version.label], - } - ); - - const pullRequests = resp.data?.repository?.pullRequests; - if (!pullRequests) { - throw new Error(`unexpected github response, unable to fetch PRs: ${inspect(resp)}`); - } - - hasNextPage = pullRequests.pageInfo?.hasNextPage; - nextCursor = pullRequests.pageInfo?.endCursor; - - if (hasNextPage === undefined || (hasNextPage && !nextCursor)) { - throw new Error( - `github response does not include valid pagination information: ${inspect(resp)}` - ); - } - - for (const node of pullRequests.nodes) { - yield parsePullRequestNode(node); - } - } -} - -export async function getPr(token: string, number: number) { - const resp = await gqlRequest( - token, - gql` - query($number: Int!) { - repository(owner: "elastic", name: "kibana") { - pullRequest(number: $number) { - ...PrNode - } - } - } - ${PrNodeFragment} - `, - { - number, - } - ); - - const node = resp.data?.repository?.pullRequest; - if (!node) { - throw new Error(`unexpected github response, unable to fetch PR: ${inspect(resp)}`); - } - - return parsePullRequestNode(node); -}