Add standard library support (#139)

Add standard library support; Add standard macros for ERC20, ERC721 and Ownable

package.json

@@ -29,7 +29,7 @@
         "build-import-parser": "tspegjs -o src/rewriter/import_directive_parser.ts --custom-header-file src/rewriter/import_directive_header.ts --cache src/rewriter/import_directive.pegjs",
         "build-parsers": "npm run build-expr-parser && npm run build-import-parser",
         "transpile": "tsc",
-        "build": "npm run clean && npm run build-parsers && npm run transpile",
+        "build": "npm run clean && npm run build-parsers && npm run transpile && cp -r stdlib/ dist/stdlib/",
         "test": "NODE_OPTIONS='--max-old-space-size=2048' nyc mocha",
         "test:ci": "NODE_OPTIONS='--max-old-space-size=2048' nyc mocha --parallel --jobs 4",
         "coverage:upload": "nyc report --reporter=text-lcov > coverage.lcov && codecov -t $CODECOV_TOKEN",

src/bin/scribble.ts

@@ -740,9 +740,15 @@ if ("version" in options) {
     // First load any macros if `--macro-path` was specified
     const macros = new Map<string, MacroDefinition>();
 
+    const macroPaths: string[] = [join(__dirname, "..", "stdlib")];
+
     if (options["macro-path"]) {
+        macroPaths.push(options["macro-path"]);
+    }
+
+    for (const macroPath of macroPaths) {
         try {
-            detectMacroDefinitions(options["macro-path"], macros, contentsMap);
+            detectMacroDefinitions(macroPath, macros, contentsMap);
         } catch (e) {
             if (e instanceof YamlSchemaError) {
                 prettyError(e.constructor.name, e.message, e.range);
@@ -887,7 +893,6 @@ if ("version" in options) {
         contentsMap,
         compilerVersionUsed,
         debugEvents,
-        new Map(),
         outputMode,
         typeEnv,
         semMap,

src/instrumenter/annotations.ts

@@ -463,6 +463,62 @@ export function gatherContractAnnotations(
 
 export class MacroError extends AnnotationError {}
 
+/**
+ * Given a parsed property P, from a macro definition, and the scope in which that macro definition is
+ * expanded, find the actual AnnotationTarget of the property P.
+ */
+function getMacroPropertyTarget(
+    scope: AnnotationTarget,
+    name: string,
+    args: string[],
+    meta: MacroMetaData,
+    ctx: AnnotationExtractionContext
+): AnnotationTarget {
+    if (name === "<contract>") {
+        assert(
+            scope instanceof ContractDefinition,
+            `Macro annotation ${meta.parsedAnnot.name} added on non-contract node ${meta.targetName}`
+        );
+        return scope;
+    }
+
+    let targets = [...resolveAny(name, scope, ctx.compilerVersion, true)];
+
+    if (targets.length > 1) {
+        targets = targets.filter((target) => {
+            // TODO: Add support for public getters
+            return (
+                target instanceof FunctionDefinition &&
+                target.vParameters.vParameters.length == args.length
+            );
+        });
+    }
+
+    if (targets.length === 0) {
+        throw new MacroError(
+            `No target ${name} found in contract ${(scope as ContractDefinition).name} for ${
+                meta.original
+            }`,
+            meta.original,
+            meta.parsedAnnot.src as Range,
+            meta.target
+        );
+    }
+
+    if (targets.length > 1) {
+        throw new MacroError(
+            `Multiple possible targets ${name} found in contract ${
+                (scope as ContractDefinition).name
+            } for ${meta.original}`,
+            meta.original,
+            meta.parsedAnnot.src as Range,
+            meta.target
+        );
+    }
+
+    return targets[0];
+}
+
 /**
  * Detects macro annotations, produces annotations that are defined by macro
  * and injects them target nodes. Macro annotations are removed afterwards.
@@ -502,20 +558,13 @@ function processMacroAnnotations(
 
                 const localAliases = new Map(globalAliases);
 
-                const targets = [...resolveAny(name, scope, ctx.compilerVersion, true)];
-
-                if (targets.length !== 1) {
-                    throw new MacroError(
-                        `No target ${name} found in contract ${
-                            (scope as ContractDefinition).name
-                        } for ${meta.original}`,
-                        meta.original,
-                        meta.parsedAnnot.src as Range,
-                        meta.target
-                    );
-                }
-
-                const target = targets[0];
+                const target: AnnotationTarget = getMacroPropertyTarget(
+                    scope,
+                    name,
+                    args,
+                    meta,
+                    ctx
+                );
 
                 if (target instanceof FunctionDefinition && args.length > 0) {
                     const params = target.vParameters.vParameters;

src/instrumenter/instrument.ts

@@ -32,7 +32,7 @@ import {
     TypeNode,
     UncheckedBlock
 } from "solc-typed-ast";
-import { AnnotationType, SId, SNode } from "../spec-lang/ast";
+import { AnnotationType, SNode } from "../spec-lang/ast";
 import {
     filterByType,
     isChangingState,
@@ -280,29 +280,22 @@ function getDebugInfoEmits(
     const instrCtx = transCtx.instrCtx;
 
     for (const annot of annotations) {
-        const dbgIdsMap = transCtx.dbgInfo.get(annot);
-
-        const evtArgs: Expression[] = [];
-        const typeList: Array<[SId, TypeNode]> = [];
-
-        // Walk over the debug id map for the current annotation and add any ids found to `evtArgs` and `typeList`.
-        for (const [, [ids, transpiledId, vType]] of dbgIdsMap.entries()) {
-            evtArgs.push(transpiledId);
-
-            // Note: This works only for primitive types. If we ever allow more complex types, the builtin
-            // `pp()` function for those may differ from the typeString that solc expects.
-            typeList.push([ids[0], vType]);
-        }
+        const dbgIdsMap = transCtx.annotationDebugMap.get(annot);
 
         // If there are no debug ids for the current annotation, there is no debug event to build
-        if (evtArgs.length == 0) {
+        if (dbgIdsMap.size() == 0) {
             res.push(undefined);
 
             continue;
         }
 
-        if (!instrCtx.debugEventsEncoding.has(annot.id)) {
-            instrCtx.debugEventsEncoding.set(annot.id, dbgIdsMap);
+        const evtArgs: Expression[] = [...dbgIdsMap.values()].map((v) => v[1]);
+
+        if (!instrCtx.debugEventsDescMap.has(annot)) {
+            instrCtx.debugEventsDescMap.set(
+                annot,
+                [...dbgIdsMap.values()].map((v) => [v[0], v[2]])
+            );
         }
 
         const assertionFailedDataEvtDef = instrCtx.getAssertionFailedDataEvent(annot.target);

src/instrumenter/instrumentation_context.ts

@@ -20,7 +20,7 @@ import {
     VariableDeclaration
 } from "solc-typed-ast";
 import { print } from "../ast_to_source_printer";
-import { SUserFunctionDefinition } from "../spec-lang/ast";
+import { SId, SUserFunctionDefinition } from "../spec-lang/ast";
 import { SemMap, TypeEnv } from "../spec-lang/tc";
 import { dedup, single } from "../util/misc";
 import { NameGenerator } from "../util/name_generator";
@@ -39,7 +39,7 @@ import {
 } from "./custom_maps_templates";
 import { makeArraySumFun, UnsupportedConstruct } from "./instrument";
 import { findAliasedStateVars } from "./state_vars";
-import { DbgIdsMap, InstrumentationSiteType, TranspilingContext } from "./transpiling_context";
+import { InstrumentationSiteType, TranspilingContext } from "./transpiling_context";
 import { FactoryMap, ScribbleFactory, StructMap } from "./utils";
 
 /**
@@ -288,6 +288,12 @@ export class InstrumentationContext {
      * Map containing debug event associated with a given annotation.
      */
     public readonly debugEventsMap: Map<AnnotationMetaData, EventDefinition> = new Map();
+    /**
+     * Map from an annotation to an array describing the arguments to the debug
+     * event emitted for this annotation.
+     */
+    public readonly debugEventsDescMap: Map<AnnotationMetaData, Array<[SId[], TypeNode]>> =
+        new Map();
 
     /**
      * Bit of a hack - this is set by `generateUtilsContract`. We need an
@@ -375,7 +381,6 @@ export class InstrumentationContext {
         public readonly files: SourceMap,
         public readonly compilerVersion: string,
         public readonly debugEvents: boolean,
-        public readonly debugEventsEncoding: Map<number, DbgIdsMap>,
         public readonly outputMode: "files" | "flat" | "json",
         public readonly typeEnv: TypeEnv,
         public readonly semMap: SemMap,

src/instrumenter/interpose.ts

@@ -135,26 +135,47 @@ function changeDependentsMutabilty(
     ctx: InstrumentationContext,
     skipStartingFun = false
 ) {
-    const queue = [fun];
+    const queue: Array<[FunctionDefinition, boolean]> = [[fun, false]];
     const seen = new Set<FunctionDefinition>();
+
     // Walk back recursively through all callers, overriden and overriding functions of fun, and
     // mark all of those that have view/pure mutability to be modified.
+    // Note that in the queue we keep track if the dependence is call-graph based, or inheritance based.
+    // For call-graph dependences, we ignore the 'skipStartingFun' flag.
+    //
+    // TODO(dimo): This whole logic with `skipStartingFun` is somewhat hacky, and needs to be simplified.
+    // I believe we had it originally to try and keep the private interposed original function as
+    // 'view' where possible, to minimize the impact on other view functions that may call it internally.
+    // However the extra small reduction in code modification may not be worth the code complexity in the long term,
+    // especially when mstore mode is also an option for those cases.
     while (queue.length > 0) {
-        const cur = queue.shift() as FunctionDefinition;
+        const [cur, callDependency] = queue.shift() as [FunctionDefinition, boolean];
 
         if (isChangingState(cur) || seen.has(cur)) {
             continue;
         }
 
-        if (cur !== fun || !skipStartingFun) {
+        if (cur !== fun || !skipStartingFun || callDependency) {
             seen.add(cur);
 
             cur.stateMutability = FunctionStateMutability.NonPayable;
         }
 
-        queue.push(...(ctx.callgraph.callers.get(cur) as FunSet));
-        queue.push(...(ctx.callgraph.overridenBy.get(cur) as FunSet));
-        queue.push(...(ctx.callgraph.overrides.get(cur) as FunSet));
+        queue.push(
+            ...[...(ctx.callgraph.callers.get(cur) as FunSet)].map(
+                (fun) => [fun, true] as [FunctionDefinition, boolean]
+            )
+        );
+        queue.push(
+            ...[...(ctx.callgraph.overridenBy.get(cur) as FunSet)].map(
+                (fun) => [fun, false] as [FunctionDefinition, boolean]
+            )
+        );
+        queue.push(
+            ...[...(ctx.callgraph.overrides.get(cur) as FunSet)].map(
+                (fun) => [fun, false] as [FunctionDefinition, boolean]
+            )
+        );
     }
 }
 

src/instrumenter/transpile.ts

@@ -211,10 +211,25 @@ function addTracingInfo(id: SId, transpiledExpr: Expression, ctx: TranspilingCon
         return;
     }
 
-    const idDebugMap = ctx.dbgInfo.get(ctx.curAnnotation);
+    /**
+     * funDebugMap contains the debug ids for ALL annotations in the current function
+     */
+    const funDebugMap = ctx.dbIdsMap;
+    /**
+     * annotDebugMap contains only the debug ids for the currently transpiled annotation
+     */
+    const annotDebugMap = ctx.annotationDebugMap.get(ctx.curAnnotation);
     const defSite = id.defSite as VarDefSite;
 
-    if (!idDebugMap.has(defSite)) {
+    let argValue: Expression;
+
+    /**
+     * If we have already computed the value of this `SId` for debugging in this function, then re-use it
+     *
+     * TODO: This whole logic should be replaced by some form of 'available expressions' analysis attached to the
+     * TranspilingContext.
+     */
+    if (!funDebugMap.has(defSite)) {
         const isOld = (ctx.semInfo.get(id) as SemInfo).isOld;
 
         if (isOld) {
@@ -231,14 +246,28 @@ function addTracingInfo(id: SId, transpiledExpr: Expression, ctx: TranspilingCon
 
             ctx.instrCtx.addAnnotationInstrumentation(ctx.curAnnotation, assignment);
 
-            idDebugMap.set([[id], ctx.refBinding(dbgBinding), exprT], defSite);
+            argValue = ctx.refBinding(dbgBinding);
         } else {
-            idDebugMap.set([[id], factory.copy(transpiledExpr), exprT], defSite);
+            argValue = factory.copy(transpiledExpr);
         }
+
+        funDebugMap.set([[id], argValue, exprT], defSite);
     } else {
-        const [ids] = idDebugMap.mustGet(defSite);
+        let ids: SId[];
+
+        [ids, argValue] = funDebugMap.mustGet(defSite);
         ids.push(id);
     }
+
+    /**
+     * Add this id's information to the per-annotation map used for computing
+     * event signatures later on.
+     */
+    if (!annotDebugMap.has(defSite)) {
+        annotDebugMap.set([[id], argValue, exprT], defSite);
+    } else {
+        annotDebugMap.mustGet(defSite)[0].push(id);
+    }
 }
 
 /**

src/instrumenter/transpiling_context.ts

@@ -138,7 +138,8 @@ export class TranspilingContext {
 
     public readonly contract: ContractDefinition;
 
-    public readonly dbgInfo = new AnnotationDebugMap();
+    public readonly dbIdsMap = new DbgIdsMap();
+    public readonly annotationDebugMap = new AnnotationDebugMap();
 
     public get containerContract(): ContractDefinition {
         const fun = this.containerFun;

src/instrumenter/utils.ts

@@ -407,6 +407,10 @@ export abstract class BaseStructMap<Args extends any[], KeyT extends string | nu
 
         return res;
     }
+
+    size(): number {
+        return this._cache.size;
+    }
 }
 
 export abstract class StructMap<

src/util/json_output.ts

@@ -16,7 +16,6 @@ import { getOr } from "./misc";
 import { dedup, MacroFile } from ".";
 import { PropertyMetaData } from "../instrumenter/annotations";
 import { InstrumentationContext } from "../instrumenter/instrumentation_context";
-import { DbgIdsMap } from "../instrumenter/transpiling_context";
 import { AnnotationType } from "../spec-lang/ast/declarations/annotation";
 import { NodeLocation } from "../spec-lang/ast/node";
 
@@ -282,11 +281,11 @@ function generatePropertyMap(
         const targetName = annotation.targetName;
         const filename = annotation.originalFileName;
 
-        const encodingData = ctx.debugEventsEncoding.get(annotation.id);
-        const encoding: DbgIdsMap = encodingData !== undefined ? encodingData : new DbgIdsMap();
+        let encoding = ctx.debugEventsDescMap.get(annotation);
+        encoding = encoding === undefined ? [] : encoding;
 
         const srcEncoding: Array<[string[], string]> = [];
-        for (const [, [ids, , type]] of encoding.entries()) {
+        for (const [ids, type] of encoding) {
             const srcMapList: string[] = [];
             for (const id of ids) {
                 const idSrc = rangeToSrcTriple(id.requiredRange, originalSourceList);