feat: Js Scope - Coding Logic (#216)

Adds the programming logic that detects tokens and functions and maps them to the instrumented package, ultimately generating token and function metrics.
Adds unit testing for all things Js Scope.
Reworks substitution logic to be able to take in arrays and objects.
`createSourceFileFromText` function for unit testing ease-of-use
Fixes significant bug in `getPackageData` that was preventing collection under certain circumstances.

---------

Co-authored-by: Francine Lucca <francinelucca@users.noreply.github.com>
Co-authored-by: Joe Harvey <jdharvey-ibm@users.noreply.github.com>

package.json

@@ -48,7 +48,7 @@
   "devDependencies": {
     "@commitlint/cli": "^19.2.1",
     "@commitlint/config-conventional": "^19.1.0",
-    "@ibm/telemetry-attributes-js": "^3.0.2",
+    "@ibm/telemetry-attributes-js": "^3.1.0",
     "@ibm/telemetry-config-schema": "^1.1.0",
     "@opentelemetry/api": "^1.8.0",
     "@opentelemetry/exporter-metrics-otlp-http": "^0.50.0",

sonar-project.properties

@@ -22,3 +22,7 @@ sonar.coverage.exclusions=\
 # Patterns used to exclude some files from duplication report
 sonar.cpd.exclusions=\
   **/src/test/**/*
+
+# Patterns to exclude from scanning altogether
+sonar.exclusions=\
+  src/test/__fixtures/**/*

src/main/core/anonymize/substitute-array.ts

@@ -0,0 +1,34 @@
+/*
+ * Copyright IBM Corp. 2023, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+import { Substitution } from './substitution.js'
+
+/**
+ * Substitutes values in the raw input based on the provided allowed values.
+ *
+ * @param raw - An array of values.
+ * @param allowedValues - The values to leave untouched.
+ * @returns The raw array with all non-allowed values replaced with anonymized versions of their
+ * values.
+ */
+export function substituteArray(raw: unknown[], allowedValues: unknown[]): unknown[] {
+  const subs = new Substitution()
+
+  return raw.map((value) => {
+    // Value is a string that's not safe
+    if (typeof value === 'string' && !allowedValues.includes(value)) {
+      return subs.put(value)
+    }
+
+    // Value is an object that's not null and not safe
+    if (typeof value === 'object' && value !== null && !allowedValues.includes(value)) {
+      return subs.put(value)
+    }
+
+    return value
+  })
+}

src/main/core/anonymize/substitute.ts → src/main/core/anonymize/substitute-object.ts

@@ -4,10 +4,8 @@
  * This source code is licensed under the Apache-2.0 license found in the
  * LICENSE file in the root directory of this source tree.
  */
-import { TypedKeyMap } from './typed-key-map.js'
 
-const subs = new TypedKeyMap()
-let curSub = 1
+import { Substitution } from './substitution.js'
 
 /**
  * Substitutes keys/values in the raw input based on the provided allowed keys and values.
@@ -18,40 +16,30 @@ let curSub = 1
  * @returns The raw object with all specified keys replaced with anonymized versions of their
  * values.
  */
-export function substitute<T extends Record<string, unknown>>(
+export function substituteObject<T extends Record<string, unknown>>(
   raw: T,
   allowedKeys: Array<keyof T>,
   allowedValues: unknown[]
 ): { [Property in keyof T]: T[Property] extends object ? string : T[Property] } {
+  const subs = new Substitution()
+
   const substitutedEntries = Object.entries(raw).map(([key, value]) => {
     // Key is not safe. Substitute key and value
     if (!allowedKeys.includes(key)) {
-      if (!subs.has(key)) {
-        subs.set(key, nextSub())
-      }
-      if (!subs.has(value)) {
-        subs.set(value, nextSub())
-      }
+      const newKey = subs.put(key)
+      const newVal = subs.put(value)
 
-      return [subs.get(key), subs.get(value)]
+      return [newKey, newVal]
     }
 
     // Key is safe. Value is a string that's not safe
     if (typeof value === 'string' && !allowedValues.includes(value)) {
-      if (!subs.has(value)) {
-        subs.set(value, nextSub())
-      }
-
-      return [key, subs.get(value)]
+      return [key, subs.put(value)]
     }
 
     // Key is safe. Value is an object that's not null and not safe
     if (typeof value === 'object' && value !== null && !allowedValues.includes(value)) {
-      if (!subs.has(value)) {
-        subs.set(value, nextSub())
-      }
-
-      return [key, subs.get(value)]
+      return [key, subs.put(value)]
     }
 
     // Both key and value are safe
@@ -60,7 +48,3 @@ export function substitute<T extends Record<string, unknown>>(
 
   return Object.fromEntries(substitutedEntries)
 }
-
-function nextSub() {
-  return `[redacted${curSub++}]`
-}

src/main/core/anonymize/substitution.ts

@@ -0,0 +1,36 @@
+/*
+ * Copyright IBM Corp. 2024, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+import { TypedKeyMap } from './typed-key-map.js'
+
+const subs = new TypedKeyMap()
+let curSub = 1
+
+function nextSub() {
+  return `[redacted${curSub++}]`
+}
+
+/**
+ * Encapsulates logic that tracks substitution values across a telemetry run.
+ */
+export class Substitution {
+  /**
+   * Puts a given value into the substitution map and returns its substituted/anonymized equivalent.
+   *
+   * @param key - The value to substitute.
+   * @returns An anonymized representation of the key (using substitution).
+   */
+  public put(key: unknown): string {
+    if (subs.has(key)) {
+      return subs.get(key)
+    }
+
+    const next = nextSub()
+    subs.set(key, next)
+
+    return next
+  }
+}

src/main/scopes/js/complex-value.ts

@@ -9,5 +9,5 @@
  * Object representing a complex value.
  */
 export class ComplexValue {
-  constructor(public complexValue: unknown) {}
+  constructor(public readonly complexValue: unknown) {}
 }

src/main/scopes/js/find-relevant-source-files.ts

@@ -73,17 +73,19 @@ export async function findRelevantSourceFiles(
     } while (shortestPathLength === undefined && packageTrees.length > 0)
 
     if (instrumentedInstallVersions === undefined) {
-      throw new NoInstallationFoundError(instrumentedPackage.name)
+      logger.error(new NoInstallationFoundError(instrumentedPackage.name))
+      return false
     }
 
     return instrumentedInstallVersions.some((version) => version === instrumentedPackage.version)
   })
+
   const filterData = await Promise.all(filterPromises)
 
   const results = sourceFiles.filter((_, index) => {
     return filterData[index]
   })
 
-  logger.traceEnter('', 'findRelevantSourceFiles', results)
+  logger.traceExit('', 'findRelevantSourceFiles', results)
   return results
 }

src/main/scopes/js/get-access-path.ts

@@ -0,0 +1,84 @@
+/*
+ * Copyright IBM Corp. 2024, 2024
+ *
+ * This source code is licensed under the Apache-2.0 license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+import * as ts from 'typescript'
+
+import type { Logger } from '../../core/log/logger.js'
+import { ComplexValue } from './complex-value.js'
+import { getNodeValueHandler } from './node-value-handler-map.js'
+
+type ExpressionContainingNode =
+  | ts.PropertyAccessExpression
+  | ts.ElementAccessExpression
+  | ts.CallExpression
+
+/**
+ * Constructs the access path to a given expression-containing node
+ * (PropertyAccessExpression, ElementAccessExpression or CallExpression)
+ * by reading the node's content recursively.
+ *
+ * @param node - TS node to retrieve access path for.
+ * @param sourceFile - Root AST node.
+ * @param logger - A logger instance.
+ * @returns Access path to node represented as string array (chunks).
+ */
+export default function getAccessPath(
+  node: ExpressionContainingNode,
+  sourceFile: ts.SourceFile,
+  logger: Logger
+) {
+  return computeAccessPath(node, sourceFile, [], true, logger)
+}
+
+/**
+ * Constructs the access path to a given node by reading the node's content recursively.
+ *
+ * @param node - TS node to retrieve access path for.
+ * @param sourceFile - Root AST node.
+ * @param currAccessPath - For internal use only, tracks current constructed access path.
+ * @param topLevel - For internal use only, tracks top level function call.
+ * @param logger - A logger instance.
+ * @returns Access path to node represented as string array (chunks).
+ */
+function computeAccessPath(
+  node: ts.Node,
+  sourceFile: ts.SourceFile,
+  currAccessPath: Array<string | ComplexValue>,
+  topLevel: boolean,
+  logger: Logger
+): Array<string | ComplexValue> {
+  switch (node.kind) {
+    case ts.SyntaxKind.Identifier:
+      return [...currAccessPath, (node as ts.Identifier).escapedText.toString()]
+    case ts.SyntaxKind.PropertyAccessExpression:
+      currAccessPath.push((node as ts.PropertyAccessExpression).name.escapedText.toString())
+      break
+    case ts.SyntaxKind.ElementAccessExpression: {
+      const argumentExpression = (node as ts.ElementAccessExpression).argumentExpression
+      const data = getNodeValueHandler(argumentExpression.kind, sourceFile, logger).getData(
+        argumentExpression
+      )
+      if (data !== undefined && data !== null) {
+        currAccessPath.push(data instanceof ComplexValue ? data : data.toString())
+      }
+      break
+    }
+  }
+
+  let accessPath = currAccessPath
+
+  if ('expression' in node) {
+    accessPath = computeAccessPath(
+      node.expression as ts.Node,
+      sourceFile,
+      currAccessPath,
+      false,
+      logger
+    )
+  }
+
+  return topLevel ? accessPath.reverse() : accessPath
+}