From ccd60655200028631882331d7ceb58e7728b8248 Mon Sep 17 00:00:00 2001
From: Patrick Kalita <pkalita@lbl.gov>
Date: Thu, 16 Nov 2023 15:53:25 -0800
Subject: [PATCH 1/7] Add standalone Validator class

This class directly uses the in-memory representation of the LinkML schema to perform validation on rows of data. The data is represented as an array of arrays (which is Handsontable's native format). An array of column headers is also required at validation time in order to map list elements to slots.
---
 lib/Validator.js        | 383 +++++++++++++++++++++
 package.json            |   1 +
 tests/Validator.test.js | 744 ++++++++++++++++++++++++++++++++++++++++
 yarn.lock               | 116 +++++++
 4 files changed, 1244 insertions(+)
 create mode 100644 lib/Validator.js
 create mode 100644 tests/Validator.test.js

diff --git a/lib/Validator.js b/lib/Validator.js
new file mode 100644
index 00000000..76a2662c
--- /dev/null
+++ b/lib/Validator.js
@@ -0,0 +1,383 @@
+import { Datatypes } from './utils/datatypes';
+import { validateUniqueValues } from './utils/validation';
+
+class Validator {
+  #schema;
+  #parser;
+  #targetClass;
+  #multivaluedDelimiter;
+  #valueValidatorMap;
+  #identifiers;
+  #uniqueKeys;
+  #results;
+
+  constructor(schema, multivaluedDelimiter = '; ', datatypeOptions = {}) {
+    this.#schema = schema;
+    this.#parser = new Datatypes(datatypeOptions);
+    this.#multivaluedDelimiter = multivaluedDelimiter;
+    this.#targetClass = undefined;
+  }
+
+  useTargetClass(className) {
+    const classDefinition = this.#schema.classes[className];
+    if (classDefinition === undefined) {
+      throw new Error(`No class named '${className}'`);
+    }
+
+    this.#targetClass = classDefinition;
+
+    this.#uniqueKeys = [];
+    if (classDefinition.unique_keys) {
+      this.#uniqueKeys = Object.entries(classDefinition.unique_keys).map(
+        ([name, definition]) => ({
+          ...definition,
+          unique_key_name: name,
+        })
+      );
+    }
+
+    // Technically LinkML only allows one `identifier: true` slot per class, but
+    // DataHarmonizer is being a little looser here and looking for any.
+    // See: https://linkml.io/linkml/schemas/slots.html#identifiers
+    this.#identifiers = Object.values(classDefinition.attributes)
+      .filter(
+        (attribute) => attribute.identifier && attribute.identifier === true
+      )
+      .map((attribute) => attribute.name);
+
+    this.#valueValidatorMap = new Map();
+  }
+
+  getValidatorForSlot(slot, inheritedRange) {
+    if (typeof slot === 'string' && this.#valueValidatorMap.has(slot)) {
+      return this.#valueValidatorMap.get(slot);
+    }
+
+    let slotDefinition;
+    if (typeof slot === 'string') {
+      // This assumes that the schema being passed in has gone through a
+      // materialize-attributes step
+      slotDefinition = this.#targetClass.attributes[slot];
+    } else {
+      slotDefinition = slot;
+    }
+
+    if (!slotDefinition.range && inheritedRange) {
+      slotDefinition.range = inheritedRange;
+    }
+
+    const slotType = this.#schema.types?.[slotDefinition.range];
+    const slotEnum = this.#schema.enums?.[slotDefinition.range];
+    const slotPermissibleValues = Object.values(
+      slotEnum?.permissible_values ?? {}
+    ).map((pv) => pv.text);
+
+    const anyOfValidators = (slotDefinition.any_of ?? []).map((subSlot) =>
+      this.getValidatorForSlot(subSlot, slotDefinition.range)
+    );
+    const allOfValidators = (slotDefinition.all_of ?? []).map((subSlot) =>
+      this.getValidatorForSlot(subSlot, slotDefinition.range)
+    );
+    const exactlyOneOfValidators = (slotDefinition.exactly_one_of ?? []).map(
+      (subSlot) => this.getValidatorForSlot(subSlot, slotDefinition.range)
+    );
+    const noneOfValidators = (slotDefinition.none_of ?? []).map((subSlot) =>
+      this.getValidatorForSlot(subSlot, slotDefinition.range)
+    );
+
+    /*
+     minimum_value and maximum_value don't technically apply to dates in LinkML,
+     but DataHarmonizer will check it anyway. It also supports the special value
+     `{today}` in those fields.
+
+     See:
+      * https://github.com/linkml/linkml/issues/1384
+      * https://github.com/linkml/linkml/issues/751
+      * https://github.com/linkml/linkml-model/issues/53
+    */
+    let slotMinimumValue = this.#parseMinMaxConstraint(
+      slotDefinition.minimum_value,
+      slotType?.uri
+    );
+    let slotMaximumValue = this.#parseMinMaxConstraint(
+      slotDefinition.maximum_value,
+      slotType?.uri
+    );
+
+    const validate = (value) => {
+      if (slotDefinition.required && !value) {
+        return 'This field is required';
+      }
+
+      if (slotDefinition.value_presence === 'PRESENT' && !value) {
+        return 'Value is not present';
+      } else if (slotDefinition.value_presence === 'ABSENT' && value) {
+        return 'Value is not absent';
+      }
+
+      if (!value) {
+        return;
+      }
+
+      let splitValues;
+      if (slotDefinition.multivalued) {
+        splitValues = value.split(this.#multivaluedDelimiter);
+        if (
+          slotDefinition.minimum_cardinality !== undefined &&
+          splitValues.length < slotDefinition.minimum_cardinality
+        ) {
+          return 'Too few entries';
+        }
+        if (
+          slotDefinition.maximum_cardinality !== undefined &&
+          splitValues.length > slotDefinition.maximum_cardinality
+        ) {
+          return 'Too many entries';
+        }
+      } else {
+        splitValues = [value];
+      }
+
+      for (const value of splitValues) {
+        if (slotType) {
+          const parsed = this.#parser.parse(value, slotType.uri);
+
+          if (parsed === undefined) {
+            return `Value does not match format for ${slotType.uri}`;
+          }
+
+          if (slotMinimumValue !== undefined && parsed < slotMinimumValue) {
+            return 'Value is less than minimum value';
+          }
+
+          if (slotMaximumValue !== undefined && parsed > slotMaximumValue) {
+            return 'Value is greater than maximum value';
+          }
+
+          if (
+            (slotDefinition.equals_string !== undefined &&
+              parsed !== slotDefinition.equals_string) ||
+            (slotDefinition.equals_number !== undefined &&
+              parsed !== slotDefinition.equals_number)
+          ) {
+            return 'Value does not match constant';
+          }
+
+          if (
+            slotDefinition.pattern !== undefined &&
+            !value.match(slotDefinition.pattern)
+          ) {
+            return 'Value does not match pattern';
+          }
+        }
+
+        if (slotEnum && !slotPermissibleValues.includes(value)) {
+          return 'Value is not allowed';
+        }
+
+        if (anyOfValidators.length) {
+          const results = anyOfValidators.map((fn) => fn(value));
+          const valid = results.some((result) => result === undefined);
+          if (!valid) {
+            return results.join('\n');
+          }
+        }
+
+        if (allOfValidators.length) {
+          const results = allOfValidators.map((fn) => fn(value));
+          const valid = results.every((result) => result === undefined);
+          if (!valid) {
+            return results.filter((result) => result !== undefined).join('\n');
+          }
+        }
+
+        if (exactlyOneOfValidators.length) {
+          const results = exactlyOneOfValidators.map((fn) => fn(value));
+          const valid =
+            results.filter((result) => result === undefined).length === 1;
+          if (!valid) {
+            const messages = results.filter((result) => result !== undefined);
+            if (!messages.length) {
+              return 'All expressions of exactly_one_of held';
+            } else {
+              return results
+                .filter((result) => result !== undefined)
+                .join('\n');
+            }
+          }
+        }
+
+        if (noneOfValidators.length) {
+          const results = noneOfValidators.map((fn) => fn(value));
+          const valid = results.every((result) => result !== undefined);
+          if (!valid) {
+            return 'One or more expressions of none_of held';
+          }
+        }
+      }
+    };
+
+    if (slotDefinition.name) {
+      this.#valueValidatorMap.set(slotDefinition.name, validate);
+    }
+
+    return validate;
+  }
+
+  validate(data, header) {
+    this.#results = {};
+
+    // Iterate over each row and each column performing the validation that can
+    // be performed atomically on the value in the cell according to the column's
+    // slot. Also while iterating through keep track which rows are empty for
+    // use later.
+    const nonEmptyRowNumbers = [];
+    for (let row = 0; row < data.length; row += 1) {
+      let nonEmpty = false;
+      for (let column = 0; column < data[row].length; column += 1) {
+        const slotName = header[column];
+        const valueValidator = this.getValidatorForSlot(slotName);
+        const value = data[row][column];
+        if (!nonEmpty && value != null && value !== '') {
+          nonEmptyRowNumbers.push(row);
+          nonEmpty = true;
+        }
+        const result = valueValidator(data[row][column]);
+        if (result !== undefined) {
+          this.#addResult(row, column, result);
+        }
+      }
+    }
+
+    // Validate that each column representing an identifier slot contains unique values
+    for (const identifier of this.#identifiers) {
+      const columns = [header.indexOf(identifier)];
+      this.#doUniquenessValidation(
+        columns,
+        data,
+        nonEmptyRowNumbers,
+        `Duplicate identifier not allowed`
+      );
+    }
+
+    // Validate that each group of columns representing unique_keys contains unique values
+    for (const uniqueKeysDefinition of this.#uniqueKeys) {
+      const columns = uniqueKeysDefinition.unique_key_slots.map((slotName) =>
+        header.indexOf(slotName)
+      );
+      this.#doUniquenessValidation(
+        columns,
+        data,
+        nonEmptyRowNumbers,
+        `Duplicate values for unique key ${uniqueKeysDefinition.unique_key_name} not allowed`
+      );
+    }
+
+    const rules = this.#targetClass.rules ?? [];
+    for (const rule of rules) {
+      if (rule.deactivated ) {
+        continue;
+      }
+
+      const preConditions = this.#buildSlotConditionGettersAndValidators(rule.preconditions, header);
+      if (preConditions.length === 0) {
+        continue;
+      }
+
+      const postConditions = this.#buildSlotConditionGettersAndValidators(rule.postconditions, header);
+      const elseConditions = this.#buildSlotConditionGettersAndValidators(rule.elseconditions, header);
+
+      for (let row = 0; row < data.length; row += 1) {
+        const preConditionsMet = preConditions.every(([getter, validator]) => {
+          const [, value] = getter(data[row]);
+          return validator(value) === undefined;
+        });
+        if (preConditionsMet) {
+          postConditions.forEach(([getter, validator]) => {
+            const [column, value] = getter(data[row]);
+            const result = validator(value);
+            if (result !== undefined) {
+              this.#addResult(row, column, result);
+            }
+          })
+        } else {
+          elseConditions.forEach(([getter, validator]) => {
+            const [column, value] = getter(data[row]);
+            const result = validator(value);
+            if (result !== undefined) {
+              this.#addResult(row, column, result);
+            }
+          })
+        }
+      }
+    }
+
+    return this.#results;
+  }
+
+  #addResult(row, column, message) {
+    if (this.#results === undefined) {
+      this.#results = {};
+    }
+    if (this.#results[row] === undefined) {
+      this.#results[row] = {};
+    }
+    this.#results[row][column] = message;
+  }
+
+  #parseMinMaxConstraint(value, type) {
+    let parsed;
+    if (
+      value !== undefined &&
+      (type === 'xsd:date' || type === 'xsd:dateTime')
+    ) {
+      if (value === `{today}`) {
+        parsed = new Date();
+      } else {
+        parsed = this.#parser.parse(value, type);
+      }
+    } else {
+      parsed = value;
+    }
+    return parsed;
+  }
+
+  #doUniquenessValidation(columns, data, nonEmptyRowNumbers, message) {
+    if (columns.some((col) => col < 0)) {
+      return;
+    }
+    const columnData = data
+      .filter((row, rowNumber) => nonEmptyRowNumbers.includes(rowNumber))
+      .map((row) => columns.map((column) => row[column]));
+    const isUnique = validateUniqueValues([columnData]);
+    for (let idx = 0; idx < isUnique.length; idx += 1) {
+      if (isUnique[idx]) {
+        continue;
+      }
+      const row = nonEmptyRowNumbers[idx];
+      for (const column of columns) {
+        this.#addResult(row, column, message);
+      }
+    }
+  }
+
+  #buildSlotConditionGettersAndValidators(classExpression, header) {
+    return Object.values(classExpression?.slot_conditions || {})
+      .map((slotCondition) => {
+        const column = header.indexOf(slotCondition.name);
+        if (column < 0) {
+          return;
+        }
+        const getter = (row) => [column, row[column]];
+        let inheritedRange = undefined;
+        if (!slotCondition.range) {
+          inheritedRange = this.#targetClass.attributes[slotCondition.name].range;
+        }
+        const validator = this.getValidatorForSlot(slotCondition, inheritedRange);
+        return [getter, validator]
+      })
+      .filter(v => v !== undefined);
+  }
+}
+
+export default Validator;
diff --git a/package.json b/package.json
index b235e355..c7120a56 100644
--- a/package.json
+++ b/package.json
@@ -47,6 +47,7 @@
     "@rollup/plugin-image": "^2.1.1",
     "@rollup/plugin-json": "^4.1.0",
     "@rollup/plugin-node-resolve": "^13.3.0",
+    "@types/jest": "^29.5.5",
     "bootstrap": "4.3.1",
     "copy-webpack-plugin": "^11.0.0",
     "css-loader": "^6.7.1",
diff --git a/tests/Validator.test.js b/tests/Validator.test.js
new file mode 100644
index 00000000..5ed80951
--- /dev/null
+++ b/tests/Validator.test.js
@@ -0,0 +1,744 @@
+import Validator from '../lib/Validator';
+
+const SCHEMA = {
+  id: 'test',
+  name: 'test',
+  types: {
+    string: {
+      uri: 'xsd:string',
+    },
+    normalizedString: {
+      uri: 'xsd:normalizedString',
+    },
+    token: {
+      uri: 'xsd:token',
+    },
+    integer: {
+      uri: 'xsd:integer',
+    },
+    nonNegativeInteger: {
+      uri: 'xsd:nonNegativeInteger',
+    },
+    float: {
+      uri: 'xsd:float',
+    },
+    double: {
+      uri: 'xsd:double',
+    },
+    decimal: {
+      uri: 'xsd:decimal',
+    },
+    date: {
+      uri: 'xsd:date',
+    },
+    dateTime: {
+      uri: 'xsd:dateTime',
+    },
+    time: {
+      uri: 'xsd:time',
+    },
+  },
+  classes: {
+    Test: {
+      name: 'Test',
+      attributes: {
+        a_string: {
+          name: 'a_string',
+          range: 'string',
+        },
+        a_normalized_string: {
+          name: 'a_normalized_string',
+          range: 'normalizedString',
+        },
+        a_token: {
+          name: 'a_token',
+          range: 'token',
+        },
+        an_integer: {
+          name: 'an_integer',
+          range: 'integer',
+        },
+        a_non_negative_integer: {
+          name: 'a_non_negative_integer',
+          range: 'nonNegativeInteger',
+        },
+        a_float: {
+          name: 'a_float',
+          range: 'float',
+        },
+        a_double: {
+          name: 'a_double',
+          range: 'double',
+        },
+        a_decimal: {
+          name: 'a_decimal',
+          range: 'decimal',
+        },
+        an_enum: {
+          name: 'an_enum',
+          range: 'Numbers',
+        },
+        a_date: {
+          name: 'a_date',
+          range: 'date',
+        },
+        a_datetime: {
+          name: 'a_datetime',
+          range: 'dateTime',
+        },
+        a_time: {
+          name: 'a_time',
+          range: 'time',
+        },
+        required_string: {
+          name: 'required_string',
+          range: 'string',
+          required: true,
+        },
+        a_big_number: {
+          name: 'a_big_number',
+          range: 'integer',
+          minimum_value: 100,
+        },
+        a_small_number: {
+          name: 'a_small_number',
+          range: 'integer',
+          maximum_value: 10,
+        },
+        a_medium_number: {
+          name: 'a_medium_number',
+          range: 'integer',
+          minimum_value: 25,
+          maximum_value: 75,
+        },
+        during_vancouver_olympics: {
+          name: 'during_vancouver_olympics',
+          range: 'date',
+          minimum_value: '2010-02-12',
+          maximum_value: '2010-02-28',
+        },
+        not_the_future: {
+          name: 'not_the_future',
+          range: 'date',
+          maximum_value: '{today}',
+        },
+        a_constant: {
+          name: 'a_constant',
+          range: 'string',
+          equals_string: 'const',
+        },
+        zip_code: {
+          name: 'zip_code',
+          range: 'string',
+          pattern: '\\d{5}(-\\d{4})?',
+        },
+        an_integer_or_enum: {
+          name: 'an_integer_or_enum',
+          any_of: [{ range: 'Numbers' }, { range: 'integer' }],
+        },
+        a_big_or_small_number: {
+          name: 'a_big_or_small_number',
+          range: 'integer',
+          any_of: [{ maximum_value: 10 }, { minimum_value: 100 }],
+        },
+        non_overlapping_intervals: {
+          name: 'non_overlapping_intervals',
+          range: 'integer',
+          exactly_one_of: [
+            {
+              minimum_value: 0,
+              maximum_value: 20,
+            },
+            {
+              minimum_value: 10,
+              maximum_value: 30,
+            },
+          ],
+        },
+        unsatisfiable: {
+          name: 'unsatisfiable',
+          all_of: [
+            {
+              range: 'string',
+              equals_string: 'zero',
+            },
+            {
+              range: 'integer',
+              equals_number: 0,
+            },
+          ],
+        },
+        no_bad_words: {
+          name: 'no_bad_words',
+          range: 'string',
+          none_of: [{ equals_string: 'cuss' }, { equals_string: 'swear' }],
+        },
+        many_strings: {
+          name: 'many_strings',
+          range: 'string',
+          multivalued: true,
+        },
+        many_enums: {
+          name: 'many_enums',
+          range: 'Numbers',
+          multivalued: true,
+        },
+        many_small_integers: {
+          name: 'many_small_integers',
+          range: 'integer',
+          multivalued: true,
+          maximum_value: 10,
+        },
+        just_a_few_strings: {
+          name: 'just_a_few_strings',
+          range: 'string',
+          multivalued: true,
+          minimum_cardinality: 2,
+          maximum_cardinality: 4,
+        },
+        an_identifier: {
+          name: 'an_identifier',
+          range: 'string',
+          identifier: true,
+        },
+        unique_key_part_a: {
+          name: 'unique_key_part_a',
+          range: 'string',
+        },
+        unique_key_part_b: {
+          name: 'unique_key_part_b',
+          range: 'string',
+        },
+        conditional_string: {
+          name: 'conditional_string',
+          range: 'string',
+        },
+        present_or_absent_string: {
+          name: 'conditional_string',
+          range: 'string'
+        }
+      },
+      unique_keys: {
+        a_two_part_unique_key: {
+          unique_key_slots: ['unique_key_part_a', 'unique_key_part_b'],
+        },
+      },
+      rules: [
+        {
+          description: "If a_string is either bingo or bongo then an_integer has to be >= 100 and a_float has to be <= 0",
+          preconditions: {
+            slot_conditions: {
+              a_string: {
+                name: 'a_string',
+                any_of: [
+                  { equals_string: 'bingo' },
+                  { equals_string: 'bongo' },
+                ],
+              },
+            },
+          },
+          postconditions: {
+            slot_conditions: {
+              an_integer: {
+                name: 'an_integer',
+                minimum_value: 100,
+              },
+              a_float: {
+                name: 'a_float',
+                maximum_value: 0,
+              },
+            },
+          },
+        },
+        {
+          description: "If conditional_string is 'big' then a_big_number is required, otherwise a_small_number is required",
+          preconditions: {
+            slot_conditions: {
+              conditional_string: {
+                name: 'conditional_string',
+                equals_string: 'big'
+              }
+            }
+          },
+          postconditions: {
+            slot_conditions: {
+              a_big_number: {
+                name: 'a_big_number',
+                required: true
+              }
+            }
+          },
+          elseconditions: {
+            slot_conditions: {
+              a_small_number: {
+                name: 'a_small_number',
+                required: true
+              }
+            }
+          }
+        },
+        {
+          description: "If present_or_absent_string is present, then a_big_number is required",
+          preconditions: {
+            slot_conditions: {
+              present_or_absent_string: {
+                name: 'present_or_absent_string',
+                value_presence: 'PRESENT'
+              }
+            }
+          },
+          postconditions: {
+            slot_conditions: {
+              a_big_number: {
+                name: 'a_big_number',
+                required: true
+              }
+            }
+          },
+        },
+        {
+          description: "If present_or_absent_string is absent, then a_small_number is required",
+          preconditions: {
+            slot_conditions: {
+              present_or_absent_string: {
+                name: 'present_or_absent_string',
+                value_presence: 'ABSENT'
+              }
+            }
+          },
+          postconditions: {
+            slot_conditions: {
+              a_big_number: {
+                name: 'a_small_number',
+                required: true
+              }
+            }
+          },
+        }
+      ],
+    },
+  },
+  enums: {
+    Numbers: {
+      name: 'Numbers',
+      permissible_values: {
+        one: { text: 'One' },
+        two: { text: 'Two' },
+        three: { text: 'Three' },
+      },
+    },
+  },
+};
+
+describe('Validator', () => {
+  it('should validate string types', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('a_string');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('hello')).toBeUndefined();
+    expect(fn('')).toBeUndefined();
+
+    fn = validator.getValidatorForSlot('a_normalized_string');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('hello')).toBeUndefined();
+    expect(fn('')).toBeUndefined();
+
+    fn = validator.getValidatorForSlot('a_token');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('hello')).toBeUndefined();
+    expect(fn('')).toBeUndefined();
+  });
+
+  it('should validate numeric types', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('an_integer');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('14')).toBeUndefined();
+    expect(fn('-14')).toBeUndefined();
+    expect(fn('3.1415')).toEqual('Value does not match format for xsd:integer');
+
+    fn = validator.getValidatorForSlot('a_non_negative_integer');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('14')).toBeUndefined();
+    expect(fn('-14')).toEqual(
+      'Value does not match format for xsd:nonNegativeInteger'
+    );
+    expect(fn('3.1415')).toEqual(
+      'Value does not match format for xsd:nonNegativeInteger'
+    );
+
+    fn = validator.getValidatorForSlot('a_float');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('3.1415')).toBeUndefined();
+    expect(fn('-7.1e-2')).toBeUndefined();
+    expect(fn('hello')).toEqual('Value does not match format for xsd:float');
+
+    fn = validator.getValidatorForSlot('a_double');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('3.1415')).toBeUndefined();
+    expect(fn('-7.1e-2')).toBeUndefined();
+    expect(fn('hello')).toEqual('Value does not match format for xsd:double');
+
+    fn = validator.getValidatorForSlot('a_decimal');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('3.1415')).toBeUndefined();
+    expect(fn('-0.071')).toBeUndefined();
+    expect(fn('hello')).toEqual('Value does not match format for xsd:decimal');
+  });
+
+  it('should validate enum types', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('an_enum');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('One')).toBeUndefined();
+    expect(fn('Whatever')).toEqual('Value is not allowed');
+  });
+
+  it('should validate date and time types', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('a_date');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('2022-02-22')).toBeUndefined();
+    expect(fn('whoops')).toEqual('Value does not match format for xsd:date');
+
+    fn = validator.getValidatorForSlot('a_datetime');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('2022-02-22 02:22')).toBeUndefined();
+    expect(fn('whoops')).toEqual(
+      'Value does not match format for xsd:dateTime'
+    );
+
+    fn = validator.getValidatorForSlot('a_time');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('22:23')).toBeUndefined();
+    expect(fn('whoops')).toEqual('Value does not match format for xsd:time');
+  });
+
+  it('should validate required fields', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('required_string');
+    expect(fn('correct')).toBeUndefined();
+    expect(fn('')).toEqual('This field is required');
+    expect(fn(undefined)).toEqual('This field is required');
+  });
+
+  it('should validate minimum and maximum numeric constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('a_big_number');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('123')).toBeUndefined();
+    expect(fn('99')).toEqual('Value is less than minimum value');
+
+    fn = validator.getValidatorForSlot('a_small_number');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('9')).toBeUndefined();
+    expect(fn('11')).toEqual('Value is greater than maximum value');
+
+    fn = validator.getValidatorForSlot('a_medium_number');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('50')).toBeUndefined();
+    expect(fn('11')).toEqual('Value is less than minimum value');
+    expect(fn('82')).toEqual('Value is greater than maximum value');
+  });
+
+  it('should validate minimum and maximum date constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('during_vancouver_olympics');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('2010-01-01')).toEqual('Value is less than minimum value');
+    expect(fn('2010-02-20')).toBeUndefined();
+    expect(fn('2010-03-01')).toEqual('Value is greater than maximum value');
+
+    fn = validator.getValidatorForSlot('not_the_future');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('2021-01-01')).toBeUndefined();
+    expect(fn('3000-01-01')).toEqual('Value is greater than maximum value');
+  });
+
+  it('should validate constant constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('a_constant');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('const')).toBeUndefined();
+    expect(fn('whoops')).toEqual('Value does not match constant');
+  });
+
+  it('should validate pattern constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('zip_code');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('99362')).toBeUndefined();
+    expect(fn('99362-1234')).toBeUndefined();
+    expect(fn('whatever')).toEqual('Value does not match pattern');
+  });
+
+  it('should validate any_of constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('an_integer_or_enum');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('One')).toBeUndefined();
+    expect(fn('99')).toBeUndefined();
+    expect(fn('99.99')).toEqual(
+      'Value is not allowed\nValue does not match format for xsd:integer'
+    );
+
+    fn = validator.getValidatorForSlot('a_big_or_small_number');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('9')).toBeUndefined();
+    expect(fn('101')).toBeUndefined();
+    expect(fn('50')).toEqual(
+      'Value is greater than maximum value\nValue is less than minimum value'
+    );
+    expect(fn('hello')).toEqual('Value does not match format for xsd:integer');
+  });
+
+  it('should validate all_of constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('unsatisfiable');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('zero')).toEqual('Value does not match format for xsd:integer');
+    expect(fn('0')).toEqual('Value does not match constant');
+  });
+
+  it('should validate exactly_one_of constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('non_overlapping_intervals');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('5')).toBeUndefined();
+    expect(fn('15')).toEqual('All expressions of exactly_one_of held');
+    expect(fn('25')).toBeUndefined();
+    expect(fn('35')).toEqual(
+      'Value is greater than maximum value\nValue is greater than maximum value'
+    );
+  });
+
+  it('should validate none_of constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('no_bad_words');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('this is okay')).toBeUndefined();
+    expect(fn('cuss')).toEqual('One or more expressions of none_of held');
+    expect(fn('swear')).toEqual('One or more expressions of none_of held');
+  });
+
+  it('should validate multivalued slots', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('many_strings');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('one')).toBeUndefined();
+    expect(fn('one; two; three')).toBeUndefined();
+
+    fn = validator.getValidatorForSlot('many_enums');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('One')).toBeUndefined();
+    expect(fn('One; Two; Three')).toBeUndefined();
+    expect(fn('One; whoops; Three')).toEqual('Value is not allowed');
+
+    fn = validator.getValidatorForSlot('many_small_integers');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('1')).toBeUndefined();
+    expect(fn('1; 2; 3')).toBeUndefined();
+    expect(fn('1; 2.2; 3')).toEqual(
+      'Value does not match format for xsd:integer'
+    );
+    expect(fn('1; 2; whoops')).toEqual(
+      'Value does not match format for xsd:integer'
+    );
+    expect(fn('11; 2; 3')).toEqual('Value is greater than maximum value');
+  });
+
+  it('should validate multivalued cardinality constraints', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    let fn = validator.getValidatorForSlot('just_a_few_strings');
+    expect(fn(undefined)).toBeUndefined();
+    expect(fn('one')).toEqual('Too few entries');
+    expect(fn('one; two; three')).toBeUndefined();
+    expect(fn('one; two; three; four; five')).toEqual('Too many entries');
+  });
+
+  it('should validate the columns of each row', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    const header = ['required_string', 'a_string', 'an_integer', 'an_enum'];
+
+    let data = [
+      ['hello', 'world', '1', 'One'],
+      ['', 'wassup', '2.2', 'Four'],
+    ];
+    const results = validator.validate(data, header);
+    expect(results).toEqual({
+      1: {
+        0: 'This field is required',
+        2: 'Value does not match format for xsd:integer',
+        3: 'Value is not allowed',
+      },
+    });
+  });
+
+  it('should validate the uniqueness of identifiers', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    const header = ['an_identifier'];
+    const data = [['one'], ['two'], [''], [''], [''], ['one'], ['three']];
+    const results = validator.validate(data, header);
+    expect(results).toEqual({
+      0: { 0: 'Duplicate identifier not allowed' },
+      5: { 0: 'Duplicate identifier not allowed' },
+    });
+  });
+
+  it('should validate the uniqueness of unique keys', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    const header = ['unique_key_part_a', 'unique_key_part_b'];
+    const data = [
+      ['one', 'two'],
+      ['three', 'four'],
+      ['', ''],
+      ['three', ''],
+      ['', 'four'],
+      ['', ''],
+      ['one', 'three'],
+      ['two', 'two'],
+      ['three', 'four'],
+      ['', 'four'],
+      ['three', 'three'],
+    ];
+    const results = validator.validate(data, header);
+    expect(results).toEqual({
+      1: {
+        0: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+        1: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+      },
+      4: {
+        0: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+        1: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+      },
+      8: {
+        0: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+        1: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+      },
+      9: {
+        0: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+        1: 'Duplicate values for unique key a_two_part_unique_key not allowed',
+      },
+    });
+  });
+
+  it('should validate rules with preconditions and postconditions', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    const header = ['a_string', 'an_integer', 'a_float'];
+    const data = [
+      ['whatever', '20', '20'],
+      ['bingo', '200', '-10'],
+      ['bongo', '99', '-30'],
+    ];
+    const results = validator.validate(data, header);
+    expect(results).toEqual({
+      2: {
+        1: 'Value is less than minimum value',
+      },
+    });
+  });
+
+  it('should validate rules with preconditions, postconditions, and elseconditions', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    const header = ['conditional_string', 'a_big_number', 'a_small_number'];
+    const data = [
+      ['big', '20', ''],
+      ['big', '200', ''],
+      ['big', '', '3'],
+      ['whatever', '', '30'],
+      ['whatever', '', '3'],
+      ['whatever', '200', ''],
+    ];
+    const results = validator.validate(data, header);
+    expect(results).toEqual({
+      0: {
+        1: 'Value is less than minimum value',
+      },
+      2: {
+        1: 'This field is required',
+      },
+      3: {
+        2: 'Value is greater than maximum value',
+      },
+      5: {
+        2: 'This field is required',
+      }
+    });
+  });
+
+  it('should validate value_presence: PRESENT rules', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    const header = ['present_or_absent_string', 'a_big_number'];
+    const data = [
+      ['', ''],
+      ['hello', '200'],
+      ['hello', ''],
+    ];
+    const results = validator.validate(data, header);
+    expect(results).toEqual({
+      2: {
+        1: 'This field is required',
+      },
+    });
+  });
+
+  it('should validate value_presence: ABSENT rules', () => {
+    const validator = new Validator(SCHEMA);
+    validator.useTargetClass('Test');
+
+    const header = ['present_or_absent_string', 'a_small_number'];
+    const data = [
+      ['', ''],
+      ['hello', ''],
+      ['', '5'],
+    ];
+    const results = validator.validate(data, header);
+    expect(results).toEqual({
+      0: {
+        1: 'This field is required',
+      },
+    });
+  });
+});
diff --git a/yarn.lock b/yarn.lock
index d14b9ee8..0f5cacd0 100644
--- a/yarn.lock
+++ b/yarn.lock
@@ -1110,6 +1110,13 @@
   dependencies:
     jest-get-type "^28.0.2"
 
+"@jest/expect-utils@^29.7.0":
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/@jest/expect-utils/-/expect-utils-29.7.0.tgz#023efe5d26a8a70f21677d0a1afc0f0a44e3a1c6"
+  integrity sha512-GlsNBWiFQFCVi9QVSx7f5AgMeLxe9YCCs5PuP2O2LdjDAA8Jh9eX7lA1Jq/xdXw3Wb3hyvlFNfZIfcRetSzYcA==
+  dependencies:
+    jest-get-type "^29.6.3"
+
 "@jest/expect@^28.1.3":
   version "28.1.3"
   resolved "https://registry.yarnpkg.com/@jest/expect/-/expect-28.1.3.tgz#9ac57e1d4491baca550f6bdbd232487177ad6a72"
@@ -1177,6 +1184,13 @@
   dependencies:
     "@sinclair/typebox" "^0.24.1"
 
+"@jest/schemas@^29.6.3":
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/@jest/schemas/-/schemas-29.6.3.tgz#430b5ce8a4e0044a7e3819663305a7b3091c8e03"
+  integrity sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA==
+  dependencies:
+    "@sinclair/typebox" "^0.27.8"
+
 "@jest/source-map@^28.1.2":
   version "28.1.2"
   resolved "https://registry.yarnpkg.com/@jest/source-map/-/source-map-28.1.2.tgz#7fe832b172b497d6663cdff6c13b0a920e139e24"
@@ -1239,6 +1253,18 @@
     "@types/yargs" "^17.0.8"
     chalk "^4.0.0"
 
+"@jest/types@^29.6.3":
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/@jest/types/-/types-29.6.3.tgz#1131f8cf634e7e84c5e77bab12f052af585fba59"
+  integrity sha512-u3UPsIilWKOM3F9CXtrG8LEJmNxwoCQC/XVj4IKYXvvpx7QIi/Kg1LI5uDmDpKlac62NUtX7eLjRh+jVZcLOzw==
+  dependencies:
+    "@jest/schemas" "^29.6.3"
+    "@types/istanbul-lib-coverage" "^2.0.0"
+    "@types/istanbul-reports" "^3.0.0"
+    "@types/node" "*"
+    "@types/yargs" "^17.0.8"
+    chalk "^4.0.0"
+
 "@jridgewell/gen-mapping@^0.3.0", "@jridgewell/gen-mapping@^0.3.2":
   version "0.3.3"
   resolved "https://registry.yarnpkg.com/@jridgewell/gen-mapping/-/gen-mapping-0.3.3.tgz#7e02e6eb5df901aaedb08514203b096614024098"
@@ -1418,6 +1444,11 @@
   resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.24.51.tgz#645f33fe4e02defe26f2f5c0410e1c094eac7f5f"
   integrity sha512-1P1OROm/rdubP5aFDSZQILU0vrLCJ4fvHt6EoqHEM+2D/G5MK3bIaymUKLit8Js9gbns5UyJnkP/TZROLw4tUA==
 
+"@sinclair/typebox@^0.27.8":
+  version "0.27.8"
+  resolved "https://registry.yarnpkg.com/@sinclair/typebox/-/typebox-0.27.8.tgz#6667fac16c436b5434a387a34dedb013198f6e6e"
+  integrity sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA==
+
 "@sinonjs/commons@^1.7.0":
   version "1.8.6"
   resolved "https://registry.yarnpkg.com/@sinonjs/commons/-/commons-1.8.6.tgz#80c516a4dc264c2a69115e7578d62581ff455ed9"
@@ -1596,6 +1627,14 @@
   dependencies:
     "@types/istanbul-lib-report" "*"
 
+"@types/jest@^29.5.5":
+  version "29.5.8"
+  resolved "https://registry.yarnpkg.com/@types/jest/-/jest-29.5.8.tgz#ed5c256fe2bc7c38b1915ee5ef1ff24a3427e120"
+  integrity sha512-fXEFTxMV2Co8ZF5aYFJv+YeA08RTYJfhtN5c9JSv/mFEMe+xxjufCb+PHL+bJcMs/ebPUsBu+UNTEz+ydXrR6g==
+  dependencies:
+    expect "^29.0.0"
+    pretty-format "^29.0.0"
+
 "@types/json-schema@*", "@types/json-schema@^7.0.8", "@types/json-schema@^7.0.9":
   version "7.0.15"
   resolved "https://registry.yarnpkg.com/@types/json-schema/-/json-schema-7.0.15.tgz#596a1747233694d50f6ad8a7869fcb6f56cf5841"
@@ -2924,6 +2963,11 @@ diff-sequences@^28.1.1:
   resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-28.1.1.tgz#9989dc731266dc2903457a70e996f3a041913ac6"
   integrity sha512-FU0iFaH/E23a+a718l8Qa/19bF9p06kgE0KipMOMadwa3SjnaElKzPaUC0vnibs6/B/9ni97s61mcejk8W1fQw==
 
+diff-sequences@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/diff-sequences/-/diff-sequences-29.6.3.tgz#4deaf894d11407c51efc8418012f9e70b84ea921"
+  integrity sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q==
+
 dir-glob@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-3.0.1.tgz#56dbf73d992a4a93ba1584f4534063fd2e41717f"
@@ -3301,6 +3345,17 @@ expect@^28.1.3:
     jest-message-util "^28.1.3"
     jest-util "^28.1.3"
 
+expect@^29.0.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/expect/-/expect-29.7.0.tgz#578874590dcb3214514084c08115d8aee61e11bc"
+  integrity sha512-2Zks0hf1VLFYI1kbh0I5jP3KHHyCHpkfyHBzsSXRFgl/Bg9mWYfMW8oD+PdMPlEwy5HNsR9JutYy6pMeOh61nw==
+  dependencies:
+    "@jest/expect-utils" "^29.7.0"
+    jest-get-type "^29.6.3"
+    jest-matcher-utils "^29.7.0"
+    jest-message-util "^29.7.0"
+    jest-util "^29.7.0"
+
 express@^4.17.3:
   version "4.18.2"
   resolved "https://registry.yarnpkg.com/express/-/express-4.18.2.tgz#3fabe08296e930c796c19e3c516979386ba9fd59"
@@ -4143,6 +4198,16 @@ jest-diff@^28.1.3:
     jest-get-type "^28.0.2"
     pretty-format "^28.1.3"
 
+jest-diff@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-diff/-/jest-diff-29.7.0.tgz#017934a66ebb7ecf6f205e84699be10afd70458a"
+  integrity sha512-LMIgiIrhigmPrs03JHpxUh2yISK3vLFPkAodPeo0+BuF7wA2FoQbkEg1u8gBYBThncu7e1oEDUfIXVuTqLRUjw==
+  dependencies:
+    chalk "^4.0.0"
+    diff-sequences "^29.6.3"
+    jest-get-type "^29.6.3"
+    pretty-format "^29.7.0"
+
 jest-docblock@^28.1.1:
   version "28.1.1"
   resolved "https://registry.yarnpkg.com/jest-docblock/-/jest-docblock-28.1.1.tgz#6f515c3bf841516d82ecd57a62eed9204c2f42a8"
@@ -4178,6 +4243,11 @@ jest-get-type@^28.0.2:
   resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-28.0.2.tgz#34622e628e4fdcd793d46db8a242227901fcf203"
   integrity sha512-ioj2w9/DxSYHfOm5lJKCdcAmPJzQXmbM/Url3rhlghrPvT3tt+7a/+oXc9azkKmLvoiXjtV83bEWqi+vs5nlPA==
 
+jest-get-type@^29.6.3:
+  version "29.6.3"
+  resolved "https://registry.yarnpkg.com/jest-get-type/-/jest-get-type-29.6.3.tgz#36f499fdcea197c1045a127319c0481723908fd1"
+  integrity sha512-zrteXnqYxfQh7l5FHyL38jL39di8H8rHoecLH3JNxH3BwOrBsNeabdap5e0I23lD4HHI8W5VFBZqG4Eaq5LNcw==
+
 jest-haste-map@^28.1.3:
   version "28.1.3"
   resolved "https://registry.yarnpkg.com/jest-haste-map/-/jest-haste-map-28.1.3.tgz#abd5451129a38d9841049644f34b034308944e2b"
@@ -4215,6 +4285,16 @@ jest-matcher-utils@^28.1.3:
     jest-get-type "^28.0.2"
     pretty-format "^28.1.3"
 
+jest-matcher-utils@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-matcher-utils/-/jest-matcher-utils-29.7.0.tgz#ae8fec79ff249fd592ce80e3ee474e83a6c44f12"
+  integrity sha512-sBkD+Xi9DtcChsI3L3u0+N0opgPYnCRPtGcQYrgXmR+hmt/fYfWAL0xRXYU8eWOdfuLgBe0YCW3AFtnRLagq/g==
+  dependencies:
+    chalk "^4.0.0"
+    jest-diff "^29.7.0"
+    jest-get-type "^29.6.3"
+    pretty-format "^29.7.0"
+
 jest-message-util@^28.1.3:
   version "28.1.3"
   resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-28.1.3.tgz#232def7f2e333f1eecc90649b5b94b0055e7c43d"
@@ -4230,6 +4310,21 @@ jest-message-util@^28.1.3:
     slash "^3.0.0"
     stack-utils "^2.0.3"
 
+jest-message-util@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-message-util/-/jest-message-util-29.7.0.tgz#8bc392e204e95dfe7564abbe72a404e28e51f7f3"
+  integrity sha512-GBEV4GRADeP+qtB2+6u61stea8mGcOT4mCtrYISZwfu9/ISHFJ/5zOMXYbpBE9RsS5+Gb63DW4FgmnKJ79Kf6w==
+  dependencies:
+    "@babel/code-frame" "^7.12.13"
+    "@jest/types" "^29.6.3"
+    "@types/stack-utils" "^2.0.0"
+    chalk "^4.0.0"
+    graceful-fs "^4.2.9"
+    micromatch "^4.0.4"
+    pretty-format "^29.7.0"
+    slash "^3.0.0"
+    stack-utils "^2.0.3"
+
 jest-mock@^28.1.3:
   version "28.1.3"
   resolved "https://registry.yarnpkg.com/jest-mock/-/jest-mock-28.1.3.tgz#d4e9b1fc838bea595c77ab73672ebf513ab249da"
@@ -4367,6 +4462,18 @@ jest-util@^28.1.3:
     graceful-fs "^4.2.9"
     picomatch "^2.2.3"
 
+jest-util@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/jest-util/-/jest-util-29.7.0.tgz#23c2b62bfb22be82b44de98055802ff3710fc0bc"
+  integrity sha512-z6EbKajIpqGKU56y5KBUgy1dt1ihhQJgWzUlZHArA/+X2ad7Cb5iF+AK1EWVL/Bo7Rz9uurpqw6SiBCefUbCGA==
+  dependencies:
+    "@jest/types" "^29.6.3"
+    "@types/node" "*"
+    chalk "^4.0.0"
+    ci-info "^3.2.0"
+    graceful-fs "^4.2.9"
+    picomatch "^2.2.3"
+
 jest-validate@^28.1.3:
   version "28.1.3"
   resolved "https://registry.yarnpkg.com/jest-validate/-/jest-validate-28.1.3.tgz#e322267fd5e7c64cea4629612c357bbda96229df"
@@ -5603,6 +5710,15 @@ pretty-format@^28.1.3:
     ansi-styles "^5.0.0"
     react-is "^18.0.0"
 
+pretty-format@^29.0.0, pretty-format@^29.7.0:
+  version "29.7.0"
+  resolved "https://registry.yarnpkg.com/pretty-format/-/pretty-format-29.7.0.tgz#ca42c758310f365bfa71a0bda0a807160b776812"
+  integrity sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ==
+  dependencies:
+    "@jest/schemas" "^29.6.3"
+    ansi-styles "^5.0.0"
+    react-is "^18.0.0"
+
 process-nextick-args@~2.0.0:
   version "2.0.1"
   resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"

From 60ee4cfa1d3d646f2dd09c590c6ca7a0cd096d9e Mon Sep 17 00:00:00 2001
From: Patrick Kalita <pkalita@lbl.gov>
Date: Wed, 22 Nov 2023 09:20:52 -0800
Subject: [PATCH 2/7] Build cache of fully induced slots when template changes;
 use more flexible caching for validation functions

---
 lib/Validator.js        | 126 +++++++++++++++++++++++++++-------------
 tests/Validator.test.js |  64 ++++++++++----------
 2 files changed, 121 insertions(+), 69 deletions(-)

diff --git a/lib/Validator.js b/lib/Validator.js
index 76a2662c..250f7fb5 100644
--- a/lib/Validator.js
+++ b/lib/Validator.js
@@ -5,6 +5,7 @@ class Validator {
   #schema;
   #parser;
   #targetClass;
+  #targetClassInducedSlots;
   #multivaluedDelimiter;
   #valueValidatorMap;
   #identifiers;
@@ -16,6 +17,7 @@ class Validator {
     this.#parser = new Datatypes(datatypeOptions);
     this.#multivaluedDelimiter = multivaluedDelimiter;
     this.#targetClass = undefined;
+    this.#targetClassInducedSlots = {};
   }
 
   useTargetClass(className) {
@@ -26,6 +28,20 @@ class Validator {
 
     this.#targetClass = classDefinition;
 
+    // We should just be able to use this.#targetClass.attributes if the JSON-formatted schema
+    // was produced with gen-linkml --materialize-attributes, but for some reason that doesn't seem
+    // to get all the slot information merged correctly. Need to look at that from the LinkML side.
+    // In the meantime, merge everything up manually.
+    this.#targetClassInducedSlots = {};
+    for (const slotName in this.#targetClass.attributes) {
+      this.#targetClassInducedSlots[slotName] = Object.assign(
+        {},
+        this.#schema.slots?.[slotName],
+        this.#targetClass.slot_usage?.[slotName],
+        this.#targetClass.attributes[slotName]
+      );
+    }
+
     this.#uniqueKeys = [];
     if (classDefinition.unique_keys) {
       this.#uniqueKeys = Object.entries(classDefinition.unique_keys).map(
@@ -39,25 +55,22 @@ class Validator {
     // Technically LinkML only allows one `identifier: true` slot per class, but
     // DataHarmonizer is being a little looser here and looking for any.
     // See: https://linkml.io/linkml/schemas/slots.html#identifiers
-    this.#identifiers = Object.values(classDefinition.attributes)
-      .filter(
-        (attribute) => attribute.identifier && attribute.identifier === true
-      )
-      .map((attribute) => attribute.name);
+    this.#identifiers = Object.values(this.#targetClassInducedSlots)
+      .filter((slot) => slot.identifier && slot.identifier === true)
+      .map((slot) => slot.name);
 
     this.#valueValidatorMap = new Map();
   }
 
-  getValidatorForSlot(slot, inheritedRange) {
-    if (typeof slot === 'string' && this.#valueValidatorMap.has(slot)) {
-      return this.#valueValidatorMap.get(slot);
+  getValidatorForSlot(slot, options = {}) {
+    const { cacheKey, inheritedRange } = options;
+    if (typeof cacheKey === 'string' && this.#valueValidatorMap.has(cacheKey)) {
+      return this.#valueValidatorMap.get(cacheKey);
     }
 
     let slotDefinition;
     if (typeof slot === 'string') {
-      // This assumes that the schema being passed in has gone through a
-      // materialize-attributes step
-      slotDefinition = this.#targetClass.attributes[slot];
+      slotDefinition = this.#targetClassInducedSlots[slot];
     } else {
       slotDefinition = slot;
     }
@@ -73,16 +86,25 @@ class Validator {
     ).map((pv) => pv.text);
 
     const anyOfValidators = (slotDefinition.any_of ?? []).map((subSlot) =>
-      this.getValidatorForSlot(subSlot, slotDefinition.range)
+      this.getValidatorForSlot(subSlot, {
+        inheritedRange: slotDefinition.range,
+      })
     );
     const allOfValidators = (slotDefinition.all_of ?? []).map((subSlot) =>
-      this.getValidatorForSlot(subSlot, slotDefinition.range)
+      this.getValidatorForSlot(subSlot, {
+        inheritedRange: slotDefinition.range,
+      })
     );
     const exactlyOneOfValidators = (slotDefinition.exactly_one_of ?? []).map(
-      (subSlot) => this.getValidatorForSlot(subSlot, slotDefinition.range)
+      (subSlot) =>
+        this.getValidatorForSlot(subSlot, {
+          inheritedRange: slotDefinition.range,
+        })
     );
     const noneOfValidators = (slotDefinition.none_of ?? []).map((subSlot) =>
-      this.getValidatorForSlot(subSlot, slotDefinition.range)
+      this.getValidatorForSlot(subSlot, {
+        inheritedRange: slotDefinition.range,
+      })
     );
 
     /*
@@ -217,8 +239,8 @@ class Validator {
       }
     };
 
-    if (slotDefinition.name) {
-      this.#valueValidatorMap.set(slotDefinition.name, validate);
+    if (typeof cacheKey === 'string') {
+      this.#valueValidatorMap.set(cacheKey, validate);
     }
 
     return validate;
@@ -227,21 +249,26 @@ class Validator {
   validate(data, header) {
     this.#results = {};
 
+    // Build a record of empty rows for later use
+    const nonEmptyRowNumbers = [];
+    for (let row = 0; row < data.length; row += 1) {
+      if (data[row].some((col) => col != null && col !== '')) {
+        nonEmptyRowNumbers.push(row);
+      }
+    }
+
     // Iterate over each row and each column performing the validation that can
     // be performed atomically on the value in the cell according to the column's
-    // slot. Also while iterating through keep track which rows are empty for
-    // use later.
-    const nonEmptyRowNumbers = [];
+    // slot.
     for (let row = 0; row < data.length; row += 1) {
-      let nonEmpty = false;
+      if (!nonEmptyRowNumbers.includes(row)) {
+        continue;
+      }
       for (let column = 0; column < data[row].length; column += 1) {
         const slotName = header[column];
-        const valueValidator = this.getValidatorForSlot(slotName);
-        const value = data[row][column];
-        if (!nonEmpty && value != null && value !== '') {
-          nonEmptyRowNumbers.push(row);
-          nonEmpty = true;
-        }
+        const valueValidator = this.getValidatorForSlot(slotName, {
+          cacheKey: slotName,
+        });
         const result = valueValidator(data[row][column]);
         if (result !== undefined) {
           this.#addResult(row, column, result);
@@ -274,18 +301,31 @@ class Validator {
     }
 
     const rules = this.#targetClass.rules ?? [];
-    for (const rule of rules) {
-      if (rule.deactivated ) {
+    for (let idx = 0; idx < rules.length; idx += 1) {
+      const rule = rules[idx];
+      if (rule.deactivated) {
         continue;
       }
 
-      const preConditions = this.#buildSlotConditionGettersAndValidators(rule.preconditions, header);
+      const preConditions = this.#buildSlotConditionGettersAndValidators(
+        rule.preconditions,
+        header,
+        `rule-${idx}-preconditions`
+      );
       if (preConditions.length === 0) {
         continue;
       }
 
-      const postConditions = this.#buildSlotConditionGettersAndValidators(rule.postconditions, header);
-      const elseConditions = this.#buildSlotConditionGettersAndValidators(rule.elseconditions, header);
+      const postConditions = this.#buildSlotConditionGettersAndValidators(
+        rule.postconditions,
+        header,
+        `rule-${idx}-postconditions`
+      );
+      const elseConditions = this.#buildSlotConditionGettersAndValidators(
+        rule.elseconditions,
+        header,
+        `rule-${idx}-elseconditions`
+      );
 
       for (let row = 0; row < data.length; row += 1) {
         const preConditionsMet = preConditions.every(([getter, validator]) => {
@@ -299,7 +339,7 @@ class Validator {
             if (result !== undefined) {
               this.#addResult(row, column, result);
             }
-          })
+          });
         } else {
           elseConditions.forEach(([getter, validator]) => {
             const [column, value] = getter(data[row]);
@@ -307,7 +347,7 @@ class Validator {
             if (result !== undefined) {
               this.#addResult(row, column, result);
             }
-          })
+          });
         }
       }
     }
@@ -361,7 +401,11 @@ class Validator {
     }
   }
 
-  #buildSlotConditionGettersAndValidators(classExpression, header) {
+  #buildSlotConditionGettersAndValidators(
+    classExpression,
+    header,
+    cachePrefix
+  ) {
     return Object.values(classExpression?.slot_conditions || {})
       .map((slotCondition) => {
         const column = header.indexOf(slotCondition.name);
@@ -371,12 +415,16 @@ class Validator {
         const getter = (row) => [column, row[column]];
         let inheritedRange = undefined;
         if (!slotCondition.range) {
-          inheritedRange = this.#targetClass.attributes[slotCondition.name].range;
+          inheritedRange =
+            this.#targetClassInducedSlots[slotCondition.name].range;
         }
-        const validator = this.getValidatorForSlot(slotCondition, inheritedRange);
-        return [getter, validator]
+        const validator = this.getValidatorForSlot(slotCondition, {
+          inheritedRange,
+          cacheKey: `${cachePrefix}-${slotCondition.name}`,
+        });
+        return [getter, validator];
       })
-      .filter(v => v !== undefined);
+      .filter((v) => v !== undefined);
   }
 }
 
diff --git a/tests/Validator.test.js b/tests/Validator.test.js
index 5ed80951..dbf76dc0 100644
--- a/tests/Validator.test.js
+++ b/tests/Validator.test.js
@@ -215,8 +215,8 @@ const SCHEMA = {
         },
         present_or_absent_string: {
           name: 'conditional_string',
-          range: 'string'
-        }
+          range: 'string',
+        },
       },
       unique_keys: {
         a_two_part_unique_key: {
@@ -225,7 +225,8 @@ const SCHEMA = {
       },
       rules: [
         {
-          description: "If a_string is either bingo or bongo then an_integer has to be >= 100 and a_float has to be <= 0",
+          description:
+            'If a_string is either bingo or bongo then an_integer has to be >= 100 and a_float has to be <= 0',
           preconditions: {
             slot_conditions: {
               a_string: {
@@ -251,70 +252,73 @@ const SCHEMA = {
           },
         },
         {
-          description: "If conditional_string is 'big' then a_big_number is required, otherwise a_small_number is required",
+          description:
+            "If conditional_string is 'big' then a_big_number is required, otherwise a_small_number is required",
           preconditions: {
             slot_conditions: {
               conditional_string: {
                 name: 'conditional_string',
-                equals_string: 'big'
-              }
-            }
+                equals_string: 'big',
+              },
+            },
           },
           postconditions: {
             slot_conditions: {
               a_big_number: {
                 name: 'a_big_number',
-                required: true
-              }
-            }
+                required: true,
+              },
+            },
           },
           elseconditions: {
             slot_conditions: {
               a_small_number: {
                 name: 'a_small_number',
-                required: true
-              }
-            }
-          }
+                required: true,
+              },
+            },
+          },
         },
         {
-          description: "If present_or_absent_string is present, then a_big_number is required",
+          description:
+            'If present_or_absent_string is present, then a_big_number is required',
           preconditions: {
             slot_conditions: {
               present_or_absent_string: {
                 name: 'present_or_absent_string',
-                value_presence: 'PRESENT'
-              }
-            }
+                value_presence: 'PRESENT',
+              },
+            },
           },
           postconditions: {
             slot_conditions: {
               a_big_number: {
                 name: 'a_big_number',
-                required: true
-              }
-            }
+                required: true,
+              },
+            },
           },
         },
         {
-          description: "If present_or_absent_string is absent, then a_small_number is required",
+          description:
+            'If present_or_absent_string is absent, then a_small_number is required',
           preconditions: {
             slot_conditions: {
               present_or_absent_string: {
                 name: 'present_or_absent_string',
-                value_presence: 'ABSENT'
-              }
-            }
+                value_presence: 'ABSENT',
+              },
+            },
           },
           postconditions: {
             slot_conditions: {
               a_big_number: {
                 name: 'a_small_number',
-                required: true
-              }
-            }
+                required: true,
+              },
+            },
           },
-        }
+        },
       ],
     },
   },
@@ -702,7 +706,7 @@ describe('Validator', () => {
       },
       5: {
         2: 'This field is required',
-      }
+      },
     });
   });
 

From 05f1e8715fc84268788d08d1b5e97a682e55106a Mon Sep 17 00:00:00 2001
From: Patrick Kalita <pkalita@lbl.gov>
Date: Tue, 28 Nov 2023 11:49:01 -0800
Subject: [PATCH 3/7] Split pre-validation repairs and actual validation; defer
 validation to Validator class

---
 lib/DataHarmonizer.js | 323 +++++++++---------------------------------
 lib/Validator.js      |   3 +-
 lib/utils/fields.js   |   2 +-
 lib/utils/general.js  |   8 ++
 4 files changed, 79 insertions(+), 257 deletions(-)

diff --git a/lib/DataHarmonizer.js b/lib/DataHarmonizer.js
index e6dede05..381e30aa 100644
--- a/lib/DataHarmonizer.js
+++ b/lib/DataHarmonizer.js
@@ -1,10 +1,10 @@
 import '@selectize/selectize';
 import Handsontable from 'handsontable';
 import $ from 'jquery';
-import { utils as XlsxUtils, read as xlsxRead } from 'xlsx/xlsx.mjs';
+import { read as xlsxRead, utils as XlsxUtils } from 'xlsx/xlsx.mjs';
 import { renderContent, urlToClickableAnchor } from './utils/content';
 
-import { wait, isValidHeaderRow } from './utils/general';
+import { isValidHeaderRow, rowIsEmpty, wait } from './utils/general';
 import { readFileAsync, updateSheetRange } from './utils/files';
 import {
   changeCase,
@@ -12,18 +12,16 @@ import {
   dataObjectToArray,
   fieldUnitBinTest,
   formatMultivaluedValue,
-  parseMultivaluedValue,
-  KEEP_ORIGINAL,
   JSON_SCHEMA_FORMAT,
+  KEEP_ORIGINAL,
+  MULTIVALUED_DELIMITER,
+  parseMultivaluedValue,
 } from './utils/fields';
-import { Datatypes } from './utils/datatypes';
 import {
   checkProvenance,
   itemCompare,
-  testNumericRange,
   validateValAgainstVocab,
   validateValsAgainstVocab,
-  validateUniqueValues,
 } from './utils/validation';
 
 import 'handsontable/dist/handsontable.full.css';
@@ -37,11 +35,12 @@ import fieldDescriptionsModal from './fieldDescriptionsModal.html';
 import HelpSidebar from './HelpSidebar';
 
 import pkg from '../package.json';
+import { DateEditor, DatetimeEditor, TimeEditor } from './editors';
+import Validator from './Validator';
+
 const VERSION = pkg.version;
 const VERSION_TEXT = 'DataHarmonizer v' + VERSION;
 
-import { DateEditor, DatetimeEditor, TimeEditor } from './editors';
-
 Handsontable.cellTypes.registerCellType('dh.datetime', {
   editor: DatetimeEditor,
   renderer: Handsontable.renderers.getRenderer('autocomplete'),
@@ -93,6 +92,11 @@ class DataHarmonizer {
     this.datetimeFormat = options.datetimeFormat || 'yyyy-MM-dd hh:mm aa';
     this.timeFormat = options.timeFormat || 'hh:mm aa';
     this.dateExportBehavior = options.dateExportBehavior || JSON_SCHEMA_FORMAT;
+    this.validator = new Validator(this.schema, MULTIVALUED_DELIMITER, {
+      dateFormat: this.dateFormat,
+      datetimeFormat: this.datetimeFormat,
+      timeFormat: this.timeFormat,
+    });
     this.self = this;
 
     // Use help sidebar by default unless turned off by client
@@ -149,6 +153,11 @@ class DataHarmonizer {
 
   useSchema(schema, export_formats, template_name) {
     this.schema = schema;
+    this.validator = new Validator(this.schema, MULTIVALUED_DELIMITER, {
+      dateFormat: this.dateFormat,
+      datetimeFormat: this.datetimeFormat,
+      timeFormat: this.timeFormat,
+    });
     this.export_formats = export_formats;
     this.useTemplate(template_name);
   }
@@ -418,6 +427,7 @@ class DataHarmonizer {
     }
 
     this.fields = this.getFields();
+    this.validator.useTargetClass(template_name);
 
     this.createHot();
   }
@@ -450,8 +460,10 @@ class DataHarmonizer {
     }
   }
 
-  validate() {
-    this.invalid_cells = this.getInvalidCells();
+  async validate() {
+    const data = this.getTrimmedData();
+    await this.doPreValidationRepairs(data);
+    this.invalid_cells = this.getInvalidCells(data);
     this.hot.render();
   }
 
@@ -547,7 +559,8 @@ class DataHarmonizer {
             if (
               Object.prototype.hasOwnProperty.call(self.invalid_cells[row], col)
             ) {
-              const msg = self.invalid_cells[row][col];
+              const msg =
+                self.invalid_cells[row][col] === 'This field is required';
               $(TD).addClass(msg ? 'empty-invalid-cell' : 'invalid-cell');
             }
           }
@@ -1457,17 +1470,11 @@ class DataHarmonizer {
    * @return {Array<Array<String>>} Grid data without trailing blank rows.
    */
   getTrimmedData() {
-    const gridData = this.hot.getData();
-    let lastEmptyRow = -1;
-    for (let i = gridData.length; i >= 0; i--) {
-      if (this.hot.isEmptyRow(i)) {
-        lastEmptyRow = i;
-      } else {
-        break;
-      }
-    }
-
-    return lastEmptyRow === -1 ? gridData : gridData.slice(0, lastEmptyRow);
+    const rowStart = 0;
+    const rowEnd = this.hot.countRows() - this.hot.countEmptyRows(true) - 1;
+    const colStart = 0;
+    const colEnd = this.hot.countCols() - 1;
+    return this.hot.getData(rowStart, colStart, rowEnd, colEnd);
   }
 
   /**
@@ -1506,7 +1513,7 @@ class DataHarmonizer {
     const listData = this.hot.getData();
     const fields = this.getFields();
     const arrayOfObjects = listData
-      .filter((row) => row.some((cell) => cell !== '' && cell != null))
+      .filter((row) => !rowIsEmpty(row))
       .map((row) =>
         dataArrayToObject(row, fields, {
           dateBehavior: this.dateExportBehavior,
@@ -2198,250 +2205,56 @@ class DataHarmonizer {
    *     message explaining why, as values. e.g,
    *     `{0: {0: 'Required cells cannot be empty'}}`
    */
-  getInvalidCells() {
-    const invalidCells = {};
-    const fields = this.getFields();
-    const columnIndex = this.getFieldYCoordinates();
-    let TODAY = new Date();
-    const datatypes = new Datatypes({
-      dateFormat: this.dateFormat,
-      datetimeFormat: this.datetimeFormat,
-      timeFormat: this.timeFormat,
-    });
-
-    let cellChanges = [];
-    const indexToRowMap = new Map();
-    const rowToIndexMap = new Map();
-    let index = 0;
-
-    let bad_pattern = {};
-
-    let full_version =
-      VERSION_TEXT +
-      ', ' +
-      this.template_name +
-      ' v' +
-      (this.template.annotations
-        ? this.template.annotations.version
-        : this.schema.version);
-
-    for (let row = 0; row < this.hot.countRows(); row++) {
-      if (this.hot.isEmptyRow(row)) continue;
-
-      // bookkeeping for non-empty rows
-      indexToRowMap.set(index, row);
-      rowToIndexMap.set(row, index);
-      index += 1;
-
-      for (let col = 0; col < fields.length; col++) {
-        const cellVal = this.hot.getDataAtCell(row, col);
-        const field = fields[col];
-        const datatype = field.datatype;
-
-        // TODO we could have messages for all types of invalidation, and add
-        //  them as tooltips
-        let msg = '';
-
-        // 1st row of provenance datatype field is forced to have a
-        // 'DataHarmonizer Version: 0.13.0' etc. value.  Change happens silently.
-        if (datatype === 'Provenance') {
-          checkProvenance(cellChanges, full_version, cellVal, row, col);
-        }
-
-        let valid = false;
+  getInvalidCells(data) {
+    const fieldNames = this.fields.map((field) => field.name);
+    return this.validator.validate(data, fieldNames);
+  }
 
-        if (!cellVal) {
-          if (field.required) {
-            msg = 'Required cells cannot be empty';
-          } else {
-            valid = true;
-          }
+  doPreValidationRepairs(data) {
+    return new Promise((resolve) => {
+      const cellChanges = [];
+      let fullVersion =
+        VERSION_TEXT +
+        ', ' +
+        this.template_name +
+        ' v' +
+        (this.template.annotations
+          ? this.template.annotations.version
+          : this.schema.version);
+
+      for (let row = 0; row < data.length; row++) {
+        if (rowIsEmpty(data[row])) {
+          continue;
         }
-
-        // If not an empty field, check its contents against field datatype AND/OR other kind of range
-        else {
-          // parseDatatype will return undefined for any value that doesn't
-          // match the pattern of the given datatype
-          const parsed = datatypes.parse(cellVal, datatype);
-          valid = parsed !== undefined;
-
-          switch (datatype) {
-            case 'xsd:integer':
-              valid &&= testNumericRange(parsed, field);
-              break;
-
-            case 'xsd:nonNegativeInteger':
-              valid &&= parsed >= 0;
-              valid &&= testNumericRange(parsed, field);
-              break;
-
-            case 'xsd:float':
-              valid &&= testNumericRange(parsed, field);
-              break;
-
-            case 'xsd:double':
-              // NEED DOUBLE RANGE VALIDATION
-              //valid = !isNaN(cellVal) && regexDouble.test(cellVal);
-              valid &&= testNumericRange(parsed, field);
-              break;
-
-            case 'xsd:decimal':
-              valid &&= testNumericRange(parsed, field);
-              break;
-
-            case 'xsd:boolean':
-              break;
-
-            case 'xsd:date':
-              valid &&= this.testDateRange(
-                cellVal,
-                field,
-                columnIndex,
-                row,
-                TODAY
-              );
-              break;
-
-            case 'xsd:string':
-              // Default: any string is valid.
-              valid = true;
-              break;
-
-            case 'xsd:normalizedString':
-              // Default: any string is valid.
-              valid = true;
-              break;
-
-            case 'xsd:token':
-              // Default: any token is valid.
-              valid = true;
-              break;
-
-            case 'Provenance':
-              // Any provenance string is valid.
-              valid = true;
-              break;
-          } // End switch
-
-          // A regular expression can be applied against a string or numeric or date value. It doesn't make sense against a categorical value.
-          if (valid && field.pattern) {
-            // Pattern shouldn't be anything other than a regular expression object
-            try {
-              valid = field.pattern.test(cellVal);
-            } catch (err) {
-              if (!(field.pattern in bad_pattern)) {
-                bad_pattern[field.pattern] = true;
-                console.log(
-                  `Regular expression /${field.pattern}/ in ${field.title} failed`,
-                  err
-                );
-              }
-              continue;
-            }
-          }
-
-          // Now perhaps value is invalid from numeric or date datatype
-          // perspective, or its an xsd:token where anything goes. Check if
-          // there are other enumeration values possible in flatVocabulary.
-          else if (
-            (!valid || datatype === 'xsd:token') &&
-            field.flatVocabulary
-          ) {
+        for (let col = 0; col < data[row].length; col++) {
+          const cellVal = data[row][col];
+          const field = this.fields[col];
+          const datatype = field.datatype;
+
+          if (datatype === 'Provenance') {
+            checkProvenance(cellChanges, fullVersion, cellVal, row, col);
+          } else if (field.flatVocabulary) {
             if (field.multivalued === true) {
-              const [vocabValid, update] = validateValsAgainstVocab(
-                cellVal,
-                field
-              );
-
-              valid = vocabValid;
+              const [, update] = validateValsAgainstVocab(cellVal, field);
               if (update) {
                 cellChanges.push([row, col, update, 'thisChange']);
               }
             } else {
-              const [vocabValid, update] = validateValAgainstVocab(
-                cellVal,
-                field
-              );
-              valid = vocabValid;
+              const [, update] = validateValAgainstVocab(cellVal, field);
               if (update) {
                 cellChanges.push([row, col, update, 'thisChange']);
               }
             }
-            // Hardcoded case: If field is xsd:token, and 1st picklist is
-            // "null value menu" then ignore validation on free-text stuff.
-            // In other words, if picklist is only being provided for null
-            // values, then field must allow free text content.
-            if (
-              !valid &&
-              field.datatype === 'xsd:token' &&
-              field.sources.length == 1 &&
-              field.sources[0] === 'null value menu'
-            ) {
-              valid = true;
-            }
-            //console.log(field.sources, field.datatype, valid, update, datatype)
-          }
-        } // End of field-not empty section
-
-        if (!valid) {
-          if (!Object.prototype.hasOwnProperty.call(invalidCells, row)) {
-            invalidCells[row] = {};
-          }
-          invalidCells[row][col] = msg;
-        }
-      } // column/field loop end
-    } // row loop end
-
-    // Check row uniqueness for identifier fields and unique_key sets
-    // This is not affected by a column's datatype.
-    const doUniqueValidation = (columnNumbers) => {
-      const values = columnNumbers.map((columnNumber) => {
-        if (columnNumber < 0) {
-          return [];
-        } else {
-          return this.hot
-            .getDataAtCol(columnNumber)
-            .filter((_, row) => rowToIndexMap.has(row));
-        }
-      });
-      const isUnique = validateUniqueValues(values);
-      isUnique.forEach((unique, uniqueIndex) => {
-        const row = indexToRowMap.get(uniqueIndex);
-        if (!unique) {
-          if (!invalidCells[row]) {
-            invalidCells[row] = {};
           }
-          columnNumbers.forEach((columnNumber) => {
-            invalidCells[row][columnNumber] = 'value must be unique';
-          });
         }
-      });
-    };
-
-    // Returns FIRST index for a field marked as an .identifier
-    const identifierFieldCol = fields.findIndex(
-      (field) => field.identifier && field.identifier === true
-    );
-    if (identifierFieldCol >= 0) {
-      doUniqueValidation([identifierFieldCol]);
-    }
-
-    // .template_unique_keys contains an object of 0 or more unique keys,
-    // each key being a combination of one or more .unique_key_slots names.
-    // Does unique validation on each unique key combo.
-    for (const unique_key of this.template_unique_keys) {
-      const uniqueKeyCols = unique_key.unique_key_slots.map((fieldName) => {
-        return fields.findIndex((field) => field.name === fieldName);
-      });
-      doUniqueValidation(uniqueKeyCols);
-    }
-
-    // Here an array of (row, column, value)... is being passed, which causes
-    // rendering operations to happen like .batch(), after all setDataAtCell()
-    // operations are completed.
-    if (cellChanges.length) this.hot.setDataAtCell(cellChanges);
-
-    return invalidCells;
+      }
+      if (cellChanges.length) {
+        this.hot.addHookOnce('afterChange', resolve);
+        this.hot.setDataAtCell(cellChanges);
+      } else {
+        resolve();
+      }
+    });
   }
 
   /**
diff --git a/lib/Validator.js b/lib/Validator.js
index 250f7fb5..bc7f4251 100644
--- a/lib/Validator.js
+++ b/lib/Validator.js
@@ -1,5 +1,6 @@
 import { Datatypes } from './utils/datatypes';
 import { validateUniqueValues } from './utils/validation';
+import { rowIsEmpty } from './utils/general';
 
 class Validator {
   #schema;
@@ -252,7 +253,7 @@ class Validator {
     // Build a record of empty rows for later use
     const nonEmptyRowNumbers = [];
     for (let row = 0; row < data.length; row += 1) {
-      if (data[row].some((col) => col != null && col !== '')) {
+      if (!rowIsEmpty(data[row])) {
         nonEmptyRowNumbers.push(row);
       }
     }
diff --git a/lib/utils/fields.js b/lib/utils/fields.js
index a0c4cf49..1165fd2f 100644
--- a/lib/utils/fields.js
+++ b/lib/utils/fields.js
@@ -7,7 +7,7 @@ import {
   datatypeIsDateOrTime,
 } from './datatypes';
 
-const MULTIVALUED_DELIMITER = '; ';
+export const MULTIVALUED_DELIMITER = '; ';
 
 /**
  * Modify a string to match specified case.
diff --git a/lib/utils/general.js b/lib/utils/general.js
index 8985f798..b526e8cd 100644
--- a/lib/utils/general.js
+++ b/lib/utils/general.js
@@ -18,3 +18,11 @@ export async function callIfFunction(value) {
 export function isValidHeaderRow(matrix, row) {
   return Number.isInteger(row) && row > 0 && row <= matrix.length;
 }
+
+/**
+ * Return true if all the values of the given array are null or an empty string
+ * @param {Array} row
+ */
+export function rowIsEmpty(row) {
+  return !row.some((col) => col != null && col !== '');
+}

From 3c6bd0fbbe77bce06d47889ce0e105e5fe291d0d Mon Sep 17 00:00:00 2001
From: Patrick Kalita <pkalita@lbl.gov>
Date: Wed, 6 Dec 2023 09:18:18 -0800
Subject: [PATCH 4/7] Fix typos in validator test schema

---
 tests/Validator.test.js | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tests/Validator.test.js b/tests/Validator.test.js
index dbf76dc0..08537fb3 100644
--- a/tests/Validator.test.js
+++ b/tests/Validator.test.js
@@ -214,7 +214,7 @@ const SCHEMA = {
           range: 'string',
         },
         present_or_absent_string: {
-          name: 'conditional_string',
+          name: 'present_or_absent_string',
           range: 'string',
         },
       },
@@ -312,7 +312,7 @@ const SCHEMA = {
           },
           postconditions: {
             slot_conditions: {
-              a_big_number: {
+              a_small_number: {
                 name: 'a_small_number',
                 required: true,
               },
@@ -326,9 +326,9 @@ const SCHEMA = {
     Numbers: {
       name: 'Numbers',
       permissible_values: {
-        one: { text: 'One' },
-        two: { text: 'Two' },
-        three: { text: 'Three' },
+        One: { text: 'One' },
+        Two: { text: 'Two' },
+        Three: { text: 'Three' },
       },
     },
   },

From d97f3d3911f1d52925bb026c6a2a59661cbcbdc5 Mon Sep 17 00:00:00 2001
From: Patrick Kalita <pkalita@lbl.gov>
Date: Wed, 6 Dec 2023 12:43:10 -0800
Subject: [PATCH 5/7] Add additional slots so that they're not reused between
 rules in validator test

---
 tests/Validator.test.js | 105 ++++++++++++++++++++++++++++------------
 1 file changed, 73 insertions(+), 32 deletions(-)

diff --git a/tests/Validator.test.js b/tests/Validator.test.js
index 08537fb3..a835d4e2 100644
--- a/tests/Validator.test.js
+++ b/tests/Validator.test.js
@@ -209,14 +209,50 @@ const SCHEMA = {
           name: 'unique_key_part_b',
           range: 'string',
         },
-        conditional_string: {
-          name: 'conditional_string',
+        rule_1_precondition_string: {
+          name: 'rule_1_precondition_string',
           range: 'string',
         },
-        present_or_absent_string: {
-          name: 'present_or_absent_string',
+        rule_1_postcondition_integer: {
+          name: 'rule_1_postcondition_integer',
+          range: 'integer',
+        },
+        rule_1_postcondition_float: {
+          name: 'rule_1_postcondition_float',
+          range: 'float',
+        },
+        rule_2_conditional_string: {
+          name: 'rule_2_conditional_string',
+          range: 'string',
+        },
+        rule_2_big_number: {
+          name: 'rule_2_big_number',
+          range: 'integer',
+          minimum_value: 100,
+        },
+        rule_2_small_number: {
+          name: 'rule_2_small_number',
+          range: 'integer',
+          maximum_value: 10,
+        },
+        rule_3_present_or_absent_string: {
+          name: 'rule_3_present_or_absent_string',
+          range: 'string',
+        },
+        rule_3_big_number: {
+          name: 'rule_3_big_number',
+          range: 'integer',
+          minimum_value: 100,
+        },
+        rule_4_present_or_absent_string: {
+          name: 'rule_4_present_or_absent_string',
           range: 'string',
         },
+        rule_4_small_number: {
+          name: 'rule_4_small_number',
+          range: 'integer',
+          maximum_value: 10,
+        },
       },
       unique_keys: {
         a_two_part_unique_key: {
@@ -225,12 +261,14 @@ const SCHEMA = {
       },
       rules: [
         {
+          title: 'rule 1',
           description:
-            'If a_string is either bingo or bongo then an_integer has to be >= 100 and a_float has to be <= 0',
+            'If rule_1_precondition_string is either bingo or bongo then rule_1_postcondition_integer ' +
+            'has to be >= 100 and rule_1_postcondition_float has to be <= 0',
           preconditions: {
             slot_conditions: {
-              a_string: {
-                name: 'a_string',
+              rule_1_precondition_string: {
+                name: 'rule_1_precondition_string',
                 any_of: [
                   { equals_string: 'bingo' },
                   { equals_string: 'bongo' },
@@ -240,80 +278,83 @@ const SCHEMA = {
           },
           postconditions: {
             slot_conditions: {
-              an_integer: {
-                name: 'an_integer',
+              rule_1_postcondition_integer: {
+                name: 'rule_1_postcondition_integer',
                 minimum_value: 100,
               },
-              a_float: {
-                name: 'a_float',
+              rule_1_postcondition_float: {
+                name: 'rule_1_postcondition_float',
                 maximum_value: 0,
               },
             },
           },
         },
         {
+          title: 'rule 2',
           description:
-            "If conditional_string is 'big' then a_big_number is required, otherwise a_small_number is required",
+            "If rule_2_conditional_string is 'big' then rule_2_big_number is required, otherwise rule_2_small_number is required",
           preconditions: {
             slot_conditions: {
-              conditional_string: {
-                name: 'conditional_string',
+              rule_2_conditional_string: {
+                name: 'rule_2_conditional_string',
                 equals_string: 'big',
               },
             },
           },
           postconditions: {
             slot_conditions: {
-              a_big_number: {
-                name: 'a_big_number',
+              rule_2_big_number: {
+                name: 'rule_2_big_number',
                 required: true,
               },
             },
           },
           elseconditions: {
             slot_conditions: {
-              a_small_number: {
-                name: 'a_small_number',
+              rule_2_small_number: {
+                name: 'rule_2_small_number',
                 required: true,
               },
             },
           },
         },
         {
+          title: 'rule 3',
           description:
-            'If present_or_absent_string is present, then a_big_number is required',
+            'If rule_3_present_or_absent_string is present, then rule_3_big_number is required',
           preconditions: {
             slot_conditions: {
-              present_or_absent_string: {
-                name: 'present_or_absent_string',
+              rule_3_present_or_absent_string: {
+                name: 'rule_3_present_or_absent_string',
                 value_presence: 'PRESENT',
               },
             },
           },
           postconditions: {
             slot_conditions: {
-              a_big_number: {
-                name: 'a_big_number',
+              rule_3_big_number: {
+                name: 'rule_3_big_number',
                 required: true,
               },
             },
           },
         },
         {
+          title: 'rule 4',
           description:
-            'If present_or_absent_string is absent, then a_small_number is required',
+            'If rule_4_present_or_absent_string is absent, then rule_4_small_number is required',
           preconditions: {
             slot_conditions: {
-              present_or_absent_string: {
-                name: 'present_or_absent_string',
+              rule_4_present_or_absent_string: {
+                name: 'rule_4_present_or_absent_string',
                 value_presence: 'ABSENT',
               },
             },
           },
           postconditions: {
             slot_conditions: {
-              a_small_number: {
-                name: 'a_small_number',
+              rule_4_small_number: {
+                name: 'rule_4_small_number',
                 required: true,
               },
             },
@@ -666,7 +707,7 @@ describe('Validator', () => {
     const validator = new Validator(SCHEMA);
     validator.useTargetClass('Test');
 
-    const header = ['a_string', 'an_integer', 'a_float'];
+    const header = ['rule_1_precondition_string', 'rule_1_postcondition_integer', 'rule_1_postcondition_float'];
     const data = [
       ['whatever', '20', '20'],
       ['bingo', '200', '-10'],
@@ -684,7 +725,7 @@ describe('Validator', () => {
     const validator = new Validator(SCHEMA);
     validator.useTargetClass('Test');
 
-    const header = ['conditional_string', 'a_big_number', 'a_small_number'];
+    const header = ['rule_2_conditional_string', 'rule_2_big_number', 'rule_2_small_number'];
     const data = [
       ['big', '20', ''],
       ['big', '200', ''],
@@ -714,7 +755,7 @@ describe('Validator', () => {
     const validator = new Validator(SCHEMA);
     validator.useTargetClass('Test');
 
-    const header = ['present_or_absent_string', 'a_big_number'];
+    const header = ['rule_3_present_or_absent_string', 'rule_3_big_number'];
     const data = [
       ['', ''],
       ['hello', '200'],
@@ -732,7 +773,7 @@ describe('Validator', () => {
     const validator = new Validator(SCHEMA);
     validator.useTargetClass('Test');
 
-    const header = ['present_or_absent_string', 'a_small_number'];
+    const header = ['rule_4_present_or_absent_string', 'rule_4_small_number'];
     const data = [
       ['', ''],
       ['hello', ''],

From 4ee8287c2fc2fa641680751811eb577a65464648 Mon Sep 17 00:00:00 2001
From: Patrick Kalita <pkalita@lbl.gov>
Date: Wed, 6 Dec 2023 12:43:59 -0800
Subject: [PATCH 6/7] Formatting

---
 tests/Validator.test.js | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/tests/Validator.test.js b/tests/Validator.test.js
index a835d4e2..3c6b4a96 100644
--- a/tests/Validator.test.js
+++ b/tests/Validator.test.js
@@ -707,7 +707,11 @@ describe('Validator', () => {
     const validator = new Validator(SCHEMA);
     validator.useTargetClass('Test');
 
-    const header = ['rule_1_precondition_string', 'rule_1_postcondition_integer', 'rule_1_postcondition_float'];
+    const header = [
+      'rule_1_precondition_string',
+      'rule_1_postcondition_integer',
+      'rule_1_postcondition_float',
+    ];
     const data = [
       ['whatever', '20', '20'],
       ['bingo', '200', '-10'],
@@ -725,7 +729,11 @@ describe('Validator', () => {
     const validator = new Validator(SCHEMA);
     validator.useTargetClass('Test');
 
-    const header = ['rule_2_conditional_string', 'rule_2_big_number', 'rule_2_small_number'];
+    const header = [
+      'rule_2_conditional_string',
+      'rule_2_big_number',
+      'rule_2_small_number',
+    ];
     const data = [
       ['big', '20', ''],
       ['big', '200', ''],

From 060b5a306f146733194630341d3233027b1f9c7f Mon Sep 17 00:00:00 2001
From: Patrick Kalita <pkalita@lbl.gov>
Date: Mon, 11 Dec 2023 10:24:00 -0800
Subject: [PATCH 7/7] Change confusing slot name in validator test

---
 tests/Validator.test.js | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tests/Validator.test.js b/tests/Validator.test.js
index 3c6b4a96..a123f587 100644
--- a/tests/Validator.test.js
+++ b/tests/Validator.test.js
@@ -141,8 +141,8 @@ const SCHEMA = {
           range: 'integer',
           any_of: [{ maximum_value: 10 }, { minimum_value: 100 }],
         },
-        non_overlapping_intervals: {
-          name: 'non_overlapping_intervals',
+        an_integer_in_exactly_one_interval: {
+          name: 'an_integer_in_exactly_one_interval',
           range: 'integer',
           exactly_one_of: [
             {
@@ -571,7 +571,9 @@ describe('Validator', () => {
     const validator = new Validator(SCHEMA);
     validator.useTargetClass('Test');
 
-    let fn = validator.getValidatorForSlot('non_overlapping_intervals');
+    let fn = validator.getValidatorForSlot(
+      'an_integer_in_exactly_one_interval'
+    );
     expect(fn(undefined)).toBeUndefined();
     expect(fn('5')).toBeUndefined();
     expect(fn('15')).toEqual('All expressions of exactly_one_of held');