Merge pull request #2 from TransitApp/feature/add-documentation

Add documentation

helpers/csv.js

@@ -73,7 +73,7 @@ function fromCsvStringToArray(string, tableName) {
       return fromCsvStringToArray(string, tableName);
     }
     process.notices.addWarning(__filename, `Row not valid in table ${tableName}: ${string}`);
-    return null;
+    return [];
   }
 
   const a = []; // Initialize array to receive values.
@@ -85,7 +85,7 @@ function fromCsvStringToArray(string, tableName) {
         /* else */
     if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"'));
     else if (m3 !== undefined) a.push(m3);
-    return ''; // Return empty string.
+    return []; // Return empty string.
   });
     // Handle special case of empty last value.
   if (/,\s*$/.test(string)) {

helpers/getters.js

@@ -2,6 +2,15 @@
 
 const schema = require('./schema');
 
+/**
+ * Get the grand parent item using one of its child.
+ *
+ * @param {Object} itemWithForeignIndexId The child item.
+ * @param {string} parentTableName        The name of the table containing the parent item.
+ * @param {string} grandParentTableName   The name of the table containing the grand parent item.
+ * @param {Gtfs} gtfs                     The GTFS containing the parent item
+ * @return {Object}                       The grand parent item.
+ */
 function getGrandParentItem(itemWithForeignIndexId, parentTableName, grandParentTableName, gtfs) {
   if (
     itemWithForeignIndexId === undefined ||
@@ -40,6 +49,14 @@ function getGrandParentItem(itemWithForeignIndexId, parentTableName, grandParent
   return gtfs.getItemWithIndexInTable(parentItem[grandParentIndexKey], grandParentTableName);
 }
 
+/**
+ * Get the child items of an item.
+ *
+ * @param {Object} parentItem     The parent item.
+ * @param {string} tableName      The name of the table containing the child items.
+ * @param {Gtfs} gtfs             The GTFS containing the child items.
+ * @return {Map.<string, Object>} Indexed child items.
+ */
 function getIndexedItemsWithParent(parentItem, tableName, gtfs) {
   if (schema.deepnessByTableName[tableName] !== 2) {
     throw new Error(`Table "${tableName}" is not of deepness 2.`);
@@ -59,6 +76,14 @@ function getIndexedItemsWithParent(parentItem, tableName, gtfs) {
   return indexedTable.get(parentItem[firstIndexKey]);
 }
 
+/**
+ * Get the child items of an item using its index.
+ *
+ * @param {Object} parentIndex    The parent item's index.
+ * @param {string} tableName      The name of the table containing the child items.
+ * @param {Gtfs} gtfs             The GTFS containing the child items.
+ * @return {Map.<string, Object>} Indexed child items.
+ */
 function getIndexedItemsWithParentIndex(parentIndex, tableName, gtfs) {
   if (schema.deepnessByTableName[tableName] !== 2) {
     throw new Error(`Table "${tableName}" is not of deepness 2.`);
@@ -72,6 +97,16 @@ function getIndexedItemsWithParentIndex(parentIndex, tableName, gtfs) {
   return indexedTable.get(parentIndex);
 }
 
+/**
+ * Get the item of a table using its index.
+ *
+ * WARNING: Will work only for the tables in which such unique indexing value exists (see schema.js for an
+ * exhaustive list)
+ *
+ * @param {string} index     Index of the item
+ * @param {string} tableName Name of the table
+ * @param {Gtfs}   gtfs      Gtfs object
+ */
 function getItemWithIndex(index, tableName, gtfs) {
   if (schema.deepnessByTableName[tableName] !== 1) {
     throw new Error(`Cannot access item with only one index in "${tableName}", since the deepness is not 1.`);
@@ -85,6 +120,17 @@ function getItemWithIndex(index, tableName, gtfs) {
   return indexedTable.get(index);
 }
 
+/**
+ * Get the item of a table using its indexes.
+ *
+ * WARNING: Will work only for the tables which have a double level of indexing, which is required when there is no
+ * value uniquely identifying each item (see schema.js for an exhaustive list)
+ *
+ * @param {string} firstIndex  First index of the item
+ * @param {string} secondIndex Second index of the item
+ * @param {string} tableName   Name of the table
+ * @param {Gtfs}   gtfs        Gtfs object
+ */
 function getItemWithIndexes(firstIndex, secondIndex, tableName, gtfs) {
   if (schema.deepnessByTableName[tableName] !== 2) {
     throw new Error(`Cannot access item with two indexes in "${tableName}", since the deep is not 2.`);
@@ -101,6 +147,14 @@ function getItemWithIndexes(firstIndex, secondIndex, tableName, gtfs) {
   return (indexedTable.has(firstIndex)) ? indexedTable.get(firstIndex).get(secondIndex) : null;
 }
 
+/**
+ * Get the parent item using one of its child.
+ *
+ * @param {Object} itemWithForeignIndexId The child item.
+ * @param {string} tableName              The name of the table containing the parent item.
+ * @param {Gtfs} gtfs                     The GTFS containing the parent item
+ * @return {Object}                       The parent item.
+ */
 function getParentItem(itemWithForeignIndexId, tableName, gtfs) {
   if (
     itemWithForeignIndexId === undefined ||

helpers/import.js

@@ -9,16 +9,22 @@ const eachWithLog = require('./logging_iterator_wrapper');
 const { fromCsvStringToArray } = require('./csv');
 const schema = require('./schema');
 
-exports.importTable = (gtfs, tableName, options) => {
-  options = options || {};
-  const indexKeys = options.indexKeys || schema.indexKeysByTableName[tableName];
+/**
+ * Import a table in the GTFS.
+ *
+ * @param {Gtfs}   gtfs      The GTFS in which to import the table.
+ * @param {string} tableName The table of the name to import.
+ */
+
+exports.importTable = (gtfs, tableName) => {
+  const indexKeys = schema.indexKeysByTableName[tableName];
   const fullPath = `${gtfs.getPath() + tableName}.txt`;
 
   if (fs.existsSync(fullPath)) {
     const fileContent = fs.readFileSync(fullPath);
-    const rows = getRows(fileContent, gtfs._regexPatternObjectsByTableName, tableName);
+    const rows = getRows(fileContent, gtfs._regexPatternObjectsByTableName.get(tableName), tableName);
 
-    gtfs._tables.set(tableName, processRows(gtfs, tableName, indexKeys, rows));
+    gtfs._tables.set(tableName, processRows(gtfs, tableName, indexKeys, rows, gtfs._shouldThrow));
     return;
   }
 
@@ -31,24 +37,22 @@ exports.importTable = (gtfs, tableName, options) => {
  * Private functions
  */
 
-function getRows(buffer, regexPatternObjectsByTableName, tableName) {
+function getRows(buffer, regexPatternObjects, tableName) {
   const rows = [];
   let rowsSlice;
   let position = 0;
   const batchLength = 50000;
   let merge;
-  const regexPatternObjects = regexPatternObjectsByTableName[tableName];
 
   while (position < buffer.length) {
     rowsSlice = buffer.toString('utf8', position, Math.min(buffer.length, position + batchLength));
 
     if (regexPatternObjects) {
-      regexPatternObjects.forEach((regexPatternObject) => {
-        const modifiedRowsSlice = rowsSlice.replace(regexPatternObject.regex, regexPatternObject.pattern || '');
+      regexPatternObjects.forEach(({regex, pattern}) => {
+        const modifiedRowsSlice = rowsSlice.replace(regex, pattern || '');
+
         if (modifiedRowsSlice !== rowsSlice) {
-          process.notices.addInfo(
-            __filename, `Applying regex replace to table: "${tableName}". regex: "${regexPatternObject.regex}".`
-          );
+          process.notices.addInfo(__filename, `Applying regex replace to table: "${tableName}". regex: "${regex}".`);
           rowsSlice = modifiedRowsSlice;
         }
       });
@@ -69,7 +73,7 @@ function getRows(buffer, regexPatternObjectsByTableName, tableName) {
   return rows;
 }
 
-function processRows(gtfs, tableName, indexKeys, rows) {
+function processRows(gtfs, tableName, indexKeys, rows, shouldThrow) {
   let table = new Map();
 
   if (rows === undefined || rows === null || rows.length === 0) {
@@ -91,6 +95,10 @@ function processRows(gtfs, tableName, indexKeys, rows) {
         }, {});
 
         if (sortedKeys.length !== arrayOfValues.length) {
+          if (shouldThrow === true) {
+            throw new Error(`Invalid raw in table ${tableName}: ${JSON.stringify(item)}`);
+          }
+
           process.notices.addWarning(__filename, `Row not valid in table: ${JSON.stringify(item)}`);
           return;
         }

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@transit/gtfs",
-  "version": "1.0.1",
+  "version": "2.0.0",
   "description": "A Node.js librairie for GTFS",
   "main": "index.js",
   "scripts": {

samples/2/stops.txt

@@ -0,0 +1,3 @@
+stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon
+stop_0,SC0,Stop 0,Some stop,37.728631,-122.431282
+stop_1,SC1,Stop 1,Some "other" stop,37.74103,-122.422482

tests.js

@@ -12,7 +12,7 @@ const { Gtfs } = require('./index');
 describe('Tests on GTFS', () => {
   // eslint-disable-next-line no-undef
   it('Test on meta functions', (done) => {
-    const path = `${__dirname}/sample/`;
+    const path = `${__dirname}/samples/1/`;
     const gtfs = new Gtfs(path);
 
     expect(gtfs.isGtfs).to.equal(true);
@@ -23,7 +23,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Test on generic table functions', (done) => {
-    const path = `${__dirname}/sample/`;
+    const path = `${__dirname}/samples/1/`;
     const gtfs = new Gtfs(path);
 
     const indexedAgencies = gtfs.getIndexedTable('agency');
@@ -80,7 +80,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on agencies', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedAgencies())).to.deep.equal(['agency_0']);
@@ -121,7 +121,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on stops', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedStops())).to.deep.equal(['stop_0', 'stop_1']);
@@ -164,7 +164,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on routes', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0']);
@@ -209,7 +209,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on trips', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedTrips())).to.deep.equal(['trip_0']);
@@ -250,7 +250,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on stop times', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedStopTimes())).to.deep.equal(['trip_0']);
@@ -314,7 +314,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on calendars', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedCalendars())).to.deep.equal(['service_0']);
@@ -359,7 +359,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on calendar dates', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedCalendarDates())).to.deep.equal(['service_0']);
@@ -420,7 +420,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on shapes', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedShapePoints())).to.deep.equal(['shape_0']);
@@ -477,7 +477,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on frequencies', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedFrequencies())).to.deep.equal(['trip_0']);
@@ -528,7 +528,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on transfers', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(sortedKeys(gtfs.getIndexedTransfers())).to.deep.equal(['stop_0', 'stop_1']);
@@ -581,7 +581,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on feed info', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     expect(gtfs.getFeedInfo().feed_lang).to.equal('en');
@@ -599,7 +599,7 @@ describe('Tests on GTFS', () => {
 
   // eslint-disable-next-line no-undef
   it('Tests on exporting', (done) => {
-    const path = `${__dirname}/sample`;
+    const path = `${__dirname}/samples/1`;
     const gtfs = new Gtfs(path);
 
     gtfs.getFeedInfo().feed_lang = 'fr';
@@ -641,6 +641,27 @@ describe('Tests on GTFS', () => {
       });
     });
   });
+
+  // eslint-disable-next-line no-undef
+  it('Tests on the regex/pattern applied to fix a bad CSV', (done) => {
+    const path = `${__dirname}/samples/2/`;
+    const gtfsWithoutFix = new Gtfs(path);
+
+    expect(() => gtfsWithoutFix.getIndexedStops()).to.throw();
+
+    const gtfsWithoutFixWithoutThrow = new Gtfs(path, { throws: false });
+
+    expect(() => gtfsWithoutFixWithoutThrow.getIndexedStops()).to.not.throw();
+
+    const regexPatternObjectsByTableName = new Map([[
+      'stops', [{regex: /,Some "other" stop,/g, pattern: ',"Some ""other"" stop",'}],
+    ]]);
+    const gtfsWithFix = new Gtfs(path, { regexPatternObjectsByTableName });
+
+    expect(gtfsWithFix.getStopWithId('stop_1').stop_desc).to.equal('Some "other" stop');
+
+    done();
+  });
 });
 
 function sortedKeys(map) {