Add TOML support with BaseTomlService (#9438)

* Add TOML support with [BaseTomlService]

Add base toml service to enable fetch of toml files
Add spec file for the new toml service for automated testing
This was added to allow a new way to retrive python version from pyproject.toml as described in issue #9410

* Fix typo

Co-authored-by: chris48s <chris48s@users.noreply.github.com>

* refactor: improve code readability

solve code review #9438 (comment)

---------

Co-authored-by: chris48s <chris48s@users.noreply.github.com>

core/base-service/base-toml.js

@@ -0,0 +1,82 @@
+/**
+ * @module
+ */
+
+import emojic from 'emojic'
+import { parse } from 'smol-toml'
+import BaseService from './base.js'
+import { InvalidResponse } from './errors.js'
+import trace from './trace.js'
+
+/**
+ * Services which query a TOML endpoint should extend BaseTomlService
+ *
+ * @abstract
+ */
+class BaseTomlService extends BaseService {
+  /**
+   * Request data from an upstream API serving TOML,
+   * parse it and validate against a schema
+   *
+   * @param {object} attrs Refer to individual attrs
+   * @param {Joi} attrs.schema Joi schema to validate the response against
+   * @param {string} attrs.url URL to request
+   * @param {object} [attrs.options={}] Options to pass to got. See
+   *    [documentation](https://github.com/sindresorhus/got/blob/main/documentation/2-options.md)
+   * @param {object} [attrs.httpErrors={}] Key-value map of status codes
+   *    and custom error messages e.g: `{ 404: 'package not found' }`.
+   *    This can be used to extend or override the
+   *    [default](https://github.com/badges/shields/blob/master/core/base-service/check-error-response.js#L5)
+   * @param {object} [attrs.systemErrors={}] Key-value map of got network exception codes
+   *    and an object of params to pass when we construct an Inaccessible exception object
+   *    e.g: `{ ECONNRESET: { prettyMessage: 'connection reset' } }`.
+   *    See {@link https://github.com/sindresorhus/got/blob/main/documentation/7-retry.md#errorcodes got error codes}
+   *    for allowed keys
+   *    and {@link module:core/base-service/errors~RuntimeErrorProps} for allowed values
+   * @returns {object} Parsed response
+   * @see https://github.com/sindresorhus/got/blob/main/documentation/2-options.md
+   */
+  async _requestToml({
+    schema,
+    url,
+    options = {},
+    httpErrors = {},
+    systemErrors = {},
+  }) {
+    const logTrace = (...args) => trace.logTrace('fetch', ...args)
+    const mergedOptions = {
+      ...{
+        headers: {
+          Accept:
+            // the official header should be application/toml - see https://toml.io/en/v1.0.0#mime-type
+            // but as this is not registered here https://www.iana.org/assignments/media-types/media-types.xhtml
+            // some apps use other mime-type like application/x-toml, text/plain etc....
+            'text/x-toml, text/toml, application/x-toml, application/toml, text/plain',
+        },
+      },
+      ...options,
+    }
+    const { buffer } = await this._request({
+      url,
+      options: mergedOptions,
+      httpErrors,
+      systemErrors,
+    })
+    let parsed
+    try {
+      parsed = parse(buffer.toString())
+    } catch (err) {
+      logTrace(emojic.dart, 'Response TOML (unparseable)', buffer)
+      throw new InvalidResponse({
+        prettyMessage: 'unparseable toml response',
+        underlyingError: err,
+      })
+    }
+    logTrace(emojic.dart, 'Response TOML (before validation)', parsed, {
+      deep: true,
+    })
+    return this.constructor._validate(parsed, schema)
+  }
+}
+
+export default BaseTomlService

core/base-service/base-toml.spec.js

@@ -0,0 +1,150 @@
+import Joi from 'joi'
+import { expect } from 'chai'
+import sinon from 'sinon'
+import BaseTomlService from './base-toml.js'
+
+const dummySchema = Joi.object({
+  requiredString: Joi.string().required(),
+}).required()
+
+class DummyTomlService extends BaseTomlService {
+  static category = 'cat'
+  static route = { base: 'foo' }
+
+  async handle() {
+    const { requiredString } = await this._requestToml({
+      schema: dummySchema,
+      url: 'http://example.com/foo.toml',
+    })
+    return { message: requiredString }
+  }
+}
+
+const expectedToml = `
+# example toml
+requiredString = "some-string"
+`
+
+const invalidSchemaToml = `
+# example toml - legal toml syntax but invalid schema
+unexpectedKey = "some-string"
+`
+
+const invalidTomlSyntax = `
+# example illegal toml syntax that can't be parsed
+missing= "space"
+colonsCantBeUsed: 42
+missing "assignment"
+`
+
+describe('BaseTomlService', function () {
+  describe('Making requests', function () {
+    let requestFetcher
+    beforeEach(function () {
+      requestFetcher = sinon.stub().returns(
+        Promise.resolve({
+          buffer: expectedToml,
+          res: { statusCode: 200 },
+        }),
+      )
+    })
+
+    it('invokes _requestFetcher', async function () {
+      await DummyTomlService.invoke(
+        { requestFetcher },
+        { handleInternalErrors: false },
+      )
+
+      expect(requestFetcher).to.have.been.calledOnceWith(
+        'http://example.com/foo.toml',
+        {
+          headers: {
+            Accept:
+              'text/x-toml, text/toml, application/x-toml, application/toml, text/plain',
+          },
+        },
+      )
+    })
+
+    it('forwards options to _requestFetcher', async function () {
+      class WithOptions extends DummyTomlService {
+        async handle() {
+          const { requiredString } = await this._requestToml({
+            schema: dummySchema,
+            url: 'http://example.com/foo.toml',
+            options: { method: 'POST', searchParams: { queryParam: 123 } },
+          })
+          return { message: requiredString }
+        }
+      }
+
+      await WithOptions.invoke(
+        { requestFetcher },
+        { handleInternalErrors: false },
+      )
+
+      expect(requestFetcher).to.have.been.calledOnceWith(
+        'http://example.com/foo.toml',
+        {
+          headers: {
+            Accept:
+              'text/x-toml, text/toml, application/x-toml, application/toml, text/plain',
+          },
+          method: 'POST',
+          searchParams: { queryParam: 123 },
+        },
+      )
+    })
+  })
+
+  describe('Making badges', function () {
+    it('handles valid toml responses', async function () {
+      const requestFetcher = async () => ({
+        buffer: expectedToml,
+        res: { statusCode: 200 },
+      })
+      expect(
+        await DummyTomlService.invoke(
+          { requestFetcher },
+          { handleInternalErrors: false },
+        ),
+      ).to.deep.equal({
+        message: 'some-string',
+      })
+    })
+
+    it('handles toml responses which do not match the schema', async function () {
+      const requestFetcher = async () => ({
+        buffer: invalidSchemaToml,
+        res: { statusCode: 200 },
+      })
+      expect(
+        await DummyTomlService.invoke(
+          { requestFetcher },
+          { handleInternalErrors: false },
+        ),
+      ).to.deep.equal({
+        isError: true,
+        color: 'lightgray',
+        message: 'invalid response data',
+      })
+    })
+
+    it('handles unparseable toml responses', async function () {
+      const requestFetcher = async () => ({
+        buffer: invalidTomlSyntax,
+        res: { statusCode: 200 },
+      })
+      expect(
+        await DummyTomlService.invoke(
+          { requestFetcher },
+          { handleInternalErrors: false },
+        ),
+      ).to.deep.equal({
+        isError: true,
+        color: 'lightgray',
+        message: 'unparseable toml response',
+      })
+    })
+  })
+})

package.json

@@ -64,6 +64,7 @@
     "query-string": "^8.1.0",
     "semver": "~7.5.4",
     "simple-icons": "9.9.0",
+    "smol-toml": "1.1.1",
     "webextension-store-meta": "^1.0.5",
     "xpath": "~0.0.33"
   },