From b5477c641562fc9475086bf726a9a05418d9f506 Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Fri, 12 Jan 2018 11:32:16 -0500 Subject: [PATCH 1/9] Draft of jsdoc for the GTFS functions, for feedback --- gtfs.js | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 77 insertions(+), 1 deletion(-) diff --git a/gtfs.js b/gtfs.js index 6621752..1c615be 100644 --- a/gtfs.js +++ b/gtfs.js @@ -11,6 +11,14 @@ const getters = require('./helpers/getters'); const { importTable } = require('./helpers/import'); const schema = require('./helpers/schema'); +/** + * Table-generic function to add items in a table of a GTFS. + * + * @param {Array} items Array of item to add to the GTFS. + * @param {string} tableName Name of the table of the GTFS in which the objects should be added. + * @param {Object} gtfs GTFS object in which to add the items. + */ + function addItems(items, tableName, gtfs) { if (items instanceof Array === false) { throw new Error(`items must be an array instead of: ${items}`); @@ -35,6 +43,16 @@ function addItems(items, tableName, gtfs) { } } +/** + * Table-generic function to get an indexed table of a GTFS. The indexation depends of the table, and is defined in + * the schema (If possible, it is the unique id). + * + * @param {string} tableName Name of the table of the GTFS to get. + * @param {Object} gtfs GTFS object containing the table to get. + * @param {Object|undefined} options Configuration object passed to importTable function. + * @return {Object} Indexed table returned + */ + function getIndexedTableOfGtfs(tableName, gtfs, options) { if (gtfs._tables.has(tableName) === false) { importTable(gtfs, tableName, options); @@ -44,9 +62,17 @@ function getIndexedTableOfGtfs(tableName, gtfs, options) { return gtfs._tables.get(tableName); } +/** + * Table-generic function to get an iterate in a table of a GTFS. + * + * @param {function} iterator Function which will be applied on every item of the table. + * @param {string} tableName Name of the table of the GTFS to enumerate. + * @param {Object} gtfs GTFS object containing the table to iterate on. + */ + function forEachItem(iterator, tableName, gtfs) { if (typeof iterator !== 'function') { - throw new Error(`iterator mulst be a function, instead of a ${typeof iterator}.`); + throw new Error(`iterator must be a function, instead of a ${typeof iterator}.`); } const indexedTable = gtfs.getIndexedTable(tableName); @@ -66,6 +92,14 @@ function forEachItem(iterator, tableName, gtfs) { } } +/** + * Table-generic function to remove items in a table of a GTFS. + * + * @param {Array} items Array of item to remove of the GTFS. + * @param {string} tableName Name of the table of the GTFS in which to add the items. + * @param {Object} gtfs GTFS object containing the table in which the object should be removed. + */ + function removeItems(items, tableName, gtfs) { if (items instanceof Array === false) { throw new Error(`items must be an array instead of: ${items}`); @@ -92,6 +126,14 @@ function removeItems(items, tableName, gtfs) { } } +/** + * Table-generic function to set an indexed table in the GTFS. + * + * @param {Object} indexedItems Object properly indexed as the schema requires the table to be. + * @param {string} tableName Name of the table of the GTFS to set. + * @param {Object} gtfs GTFS object in which the table will be set. + */ + function setIndexedItems(indexedItems, tableName, gtfs) { if (indexedItems instanceof Map === false && schema.deepnessByTableName[tableName] !== 0) { throw new Error(`indexedItems must be a Map instead of: ${indexedItems}`); @@ -101,6 +143,14 @@ function setIndexedItems(indexedItems, tableName, gtfs) { } class Gtfs { + /** + * Constructor of the GTFS + * + * @param {string} path Path to the folder contains the GTFS text files. + * @param {Object|undefined} regexPatternObjectsByTableName Optional ad-hoc regex to fix the tables. See importTable. + * @return {Object} gtfs Instanciated GTFS object. + */ + constructor(path, regexPatternObjectsByTableName) { if (typeof path !== 'string' || path.length === 0) { throw new Error(`Gtfs need a valid input path as string, instead of: "${path}".`); @@ -120,12 +170,38 @@ class Gtfs { } /* io */ + /** + * Async function exporting the GTFS at a specific path. + * + * @param {string} path Path to the folder which will contains the GTFS. The folder will be created if needed. + * @param {function} callback Function called when the export will be done. + */ exportAtPath(path, callback) { exportGtfs(this, path, callback); } + + /** + * Getter returning the path of the GTFS when it was imported. + * + * @return {string} Path to the imported GTFS. + */ getPath() { return this._path; } /* Generic table & item manipulation */ + /** + * Table-generic function to add an item in a table of the GTFS. + * + * @param {Object} item Item to add in the GTFS. + * @param {string} tableName Name of the table of the GTFS in which the item will be added. + */ addItemInTable(item, tableName) { addItems([item], tableName, this); } + + /** + * Table-generic function to add items in a table of the GTFS. + * + * @param {Array} items Array of items to add in the GTFS. + * @param {string} tableName Name of the table of the GTFS in which the item will be added. + */ addItemsInTable(items, tableName) { addItems(items, tableName, this); } + forEachItemInTable(tableName, iterator) { forEachItem(iterator, tableName, this); } forEachTableName(iterator) { this.getTableNames().forEach(iterator); } getIndexedTable(tableName, forcedValuesByKeys) { return getIndexedTableOfGtfs(tableName, this, forcedValuesByKeys); } From 58ed2e28ebc9a17d44fd50bd432979daf4844f55 Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Fri, 12 Jan 2018 13:46:22 -0500 Subject: [PATCH 2/9] Reorder argument to start with GTFS and fix comments --- gtfs.js | 211 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 107 insertions(+), 104 deletions(-) diff --git a/gtfs.js b/gtfs.js index 1c615be..eeb6d16 100644 --- a/gtfs.js +++ b/gtfs.js @@ -14,12 +14,12 @@ const schema = require('./helpers/schema'); /** * Table-generic function to add items in a table of a GTFS. * - * @param {Array} items Array of item to add to the GTFS. + * @param {Gtfs} gtfs GTFS object in which to add the items. * @param {string} tableName Name of the table of the GTFS in which the objects should be added. - * @param {Object} gtfs GTFS object in which to add the items. + * @param {Array.} items Array of items to add to the GTFS. */ -function addItems(items, tableName, gtfs) { +function addItems(gtfs, tableName, items) { if (items instanceof Array === false) { throw new Error(`items must be an array instead of: ${items}`); } @@ -45,15 +45,15 @@ function addItems(items, tableName, gtfs) { /** * Table-generic function to get an indexed table of a GTFS. The indexation depends of the table, and is defined in - * the schema (If possible, it is the unique id). + * the schema (see schema.js). * - * @param {string} tableName Name of the table of the GTFS to get. - * @param {Object} gtfs GTFS object containing the table to get. - * @param {Object|undefined} options Configuration object passed to importTable function. - * @return {Object} Indexed table returned + * @param {Gtfs} gtfs GTFS object containing the table to get. + * @param {string} tableName Name of the table of the GTFS to get. + * @param {Object} [options] Configuration object passed to importTable function. + * @return {Object} Indexed table returned */ -function getIndexedTableOfGtfs(tableName, gtfs, options) { +function getIndexedTable(gtfs, tableName, options) { if (gtfs._tables.has(tableName) === false) { importTable(gtfs, tableName, options); infoLog(`[Importation] Table ${tableName} has been imported.`); @@ -65,12 +65,12 @@ function getIndexedTableOfGtfs(tableName, gtfs, options) { /** * Table-generic function to get an iterate in a table of a GTFS. * - * @param {function} iterator Function which will be applied on every item of the table. + * @param {Gtfs} gtfs GTFS object containing the table to iterate on. * @param {string} tableName Name of the table of the GTFS to enumerate. - * @param {Object} gtfs GTFS object containing the table to iterate on. + * @param {function} iterator Function which will be applied on every item of the table. */ -function forEachItem(iterator, tableName, gtfs) { +function forEachItem(gtfs, tableName, iterator) { if (typeof iterator !== 'function') { throw new Error(`iterator must be a function, instead of a ${typeof iterator}.`); } @@ -95,12 +95,12 @@ function forEachItem(iterator, tableName, gtfs) { /** * Table-generic function to remove items in a table of a GTFS. * - * @param {Array} items Array of item to remove of the GTFS. - * @param {string} tableName Name of the table of the GTFS in which to add the items. - * @param {Object} gtfs GTFS object containing the table in which the object should be removed. + * @param {Gtfs} gtfs GTFS object containing the table in which the object should be removed. + * @param {string} tableName Name of the table of the GTFS in which to add the items. + * @param {Array.} items Array of items to remove of the GTFS. */ -function removeItems(items, tableName, gtfs) { +function removeItems(gtfs, tableName, items) { if (items instanceof Array === false) { throw new Error(`items must be an array instead of: ${items}`); } @@ -129,12 +129,12 @@ function removeItems(items, tableName, gtfs) { /** * Table-generic function to set an indexed table in the GTFS. * - * @param {Object} indexedItems Object properly indexed as the schema requires the table to be. + * @param {Gtfs} gtfs GTFS object in which the table will be set. * @param {string} tableName Name of the table of the GTFS to set. - * @param {Object} gtfs GTFS object in which the table will be set. + * @param {Map} indexedItems Object properly indexed as the schema requires the table to be (see schema.js). */ -function setIndexedItems(indexedItems, tableName, gtfs) { +function setIndexedItems(gtfs, tableName, indexedItems) { if (indexedItems instanceof Map === false && schema.deepnessByTableName[tableName] !== 0) { throw new Error(`indexedItems must be a Map instead of: ${indexedItems}`); } @@ -146,9 +146,12 @@ class Gtfs { /** * Constructor of the GTFS * - * @param {string} path Path to the folder contains the GTFS text files. - * @param {Object|undefined} regexPatternObjectsByTableName Optional ad-hoc regex to fix the tables. See importTable. - * @return {Object} gtfs Instanciated GTFS object. + * @param {string} path Path to the folder that contains the GTFS text files. + * @param {Map.< + * string, + * Array.<{regex: RegExp, pattern: string}> + * >} [regexPatternObjectsByTableName] Optional ad-hoc regex to fix the tables. See importTable. + * @return {Gtfs} gtfs Instanciated GTFS object. */ constructor(path, regexPatternObjectsByTableName) { @@ -173,7 +176,7 @@ class Gtfs { /** * Async function exporting the GTFS at a specific path. * - * @param {string} path Path to the folder which will contains the GTFS. The folder will be created if needed. + * @param {string} path Path to the folder which will contain the GTFS. The folder will be created if needed. * @param {function} callback Function called when the export will be done. */ exportAtPath(path, callback) { exportGtfs(this, path, callback); } @@ -192,108 +195,108 @@ class Gtfs { * @param {Object} item Item to add in the GTFS. * @param {string} tableName Name of the table of the GTFS in which the item will be added. */ - addItemInTable(item, tableName) { addItems([item], tableName, this); } + addItemInTable(item, tableName) { addItems(this, tableName, [item]); } /** * Table-generic function to add items in a table of the GTFS. * - * @param {Array} items Array of items to add in the GTFS. - * @param {string} tableName Name of the table of the GTFS in which the item will be added. + * @param {Array.} items Array of items to add in the GTFS. + * @param {string} tableName Name of the table of the GTFS in which the item will be added. */ - addItemsInTable(items, tableName) { addItems(items, tableName, this); } + addItemsInTable(items, tableName) { addItems(this, tableName, items); } - forEachItemInTable(tableName, iterator) { forEachItem(iterator, tableName, this); } + forEachItemInTable(tableName, iterator) { forEachItem(this, tableName, iterator); } forEachTableName(iterator) { this.getTableNames().forEach(iterator); } - getIndexedTable(tableName, forcedValuesByKeys) { return getIndexedTableOfGtfs(tableName, this, forcedValuesByKeys); } + getIndexedTable(tableName, forcedValuesByKeys) { return getIndexedTable(this, tableName, forcedValuesByKeys); } getItemWithIndexInTable(index, tableName) { return getters.getItemWithIndex(index, tableName, this); } getTableNames() { return new Set([...schema.tableNames, ...this._tables.keys()]); } getParentItem(item, tableName) { return getters.getParentItem(item, tableName, this); } - removeItemInTable(item, tableName) { removeItems([item], tableName, this); } - removeItemsInTable(items, tableName) { removeItems(items, tableName, this); } - setIndexedItemsAsTable(indexedItems, tableName) { setIndexedItems(indexedItems, tableName, this); } + removeItemInTable(item, tableName) { removeItems(this, tableName, [item]); } + removeItemsInTable(items, tableName) { removeItems(this, tableName, items); } + setIndexedItemsAsTable(indexedItems, tableName) { setIndexedItems(this, tableName, indexedItems); } /* agency.txt */ - addAgency(agency) { addItems([agency], 'agency', this); } - addAgencies(agencies) { addItems(agencies, 'agency', this); } - forEachAgency(iterator) { forEachItem(iterator, 'agency', this); } + addAgency(agency) { addItems(this, 'agency', [agency]); } + addAgencies(agencies) { addItems(this, 'agency', agencies); } + forEachAgency(iterator) { forEachItem(this, 'agency', iterator); } getAgencyOfRoute(route) { return getters.getParentItem(route, 'agency', this); } getAgencyWithId(agencyId) { return getters.getItemWithIndex(agencyId, 'agency', this); } - getIndexedAgencies() { return getIndexedTableOfGtfs('agency', this); } - removeAgency(agency) { removeItems([agency], 'agency', this); } - removeAgencies(agencies) { removeItems(agencies, 'agency', this); } - setIndexedAgencies(indexedAgencies) { setIndexedItems(indexedAgencies, 'agency', this); } + getIndexedAgencies() { return getIndexedTable(this, 'agency'); } + removeAgency(agency) { removeItems(this, 'agency', [agency]); } + removeAgencies(agencies) { removeItems(this, 'agency', agencies); } + setIndexedAgencies(indexedAgencies) { setIndexedItems(this, 'agency', indexedAgencies); } /* stops.txt */ - addStop(stop) { addItems([stop], 'stops', this); } - addStops(stops) { addItems(stops, 'stops', this); } - forEachStop(iterator) { forEachItem(iterator, 'stops', this); } - getIndexedStops() { return getIndexedTableOfGtfs('stops', this); } + addStop(stop) { addItems(this, 'stops', [stop]); } + addStops(stops) { addItems(this, 'stops', stops); } + forEachStop(iterator) { forEachItem(this, 'stops', iterator); } + getIndexedStops() { return getIndexedTable(this, 'stops'); } getStopOfStopTime(stopTime) { return getters.getParentItem(stopTime, 'stops', this); } getStopWithId(stopId) { return getters.getItemWithIndex(stopId, 'stops', this); } - removeStop(stop) { removeItems([stop], 'stops', this); } - removeStops(stops) { removeItems(stops, 'stops', this); } - setIndexedStops(indexedStops) { setIndexedItems(indexedStops, 'stops', this); } + removeStop(stop) { removeItems(this, 'stops', [stop]); } + removeStops(stops) { removeItems(this, 'stops', stops); } + setIndexedStops(indexedStops) { setIndexedItems(this, 'stops', indexedStops); } /* routes.txt */ - addRoute(route) { addItems([route], 'routes', this); } - addRoutes(routes) { addItems(routes, 'routes', this); } - forEachRoute(iterator) { forEachItem(iterator, 'routes', this); } - getIndexedRoutes() { return getIndexedTableOfGtfs('routes', this); } + addRoute(route) { addItems(this, 'routes', [route]); } + addRoutes(routes) { addItems(this, 'routes', routes); } + forEachRoute(iterator) { forEachItem(this, 'routes', iterator); } + getIndexedRoutes() { return getIndexedTable(this, 'routes'); } getRouteOfStopTime(stopTime) { return getters.getGrandParentItem(stopTime, 'trips', 'routes', this); } getRouteOfTrip(trip) { return getters.getParentItem(trip, 'routes', this); } getRouteWithId(routeId) { return getters.getItemWithIndex(routeId, 'routes', this); } - removeRoute(route) { removeItems([route], 'routes', this); } - removeRoutes(routes) { removeItems(routes, 'routes', this); } - setIndexedRoutes(indexedRoutes) { setIndexedItems(indexedRoutes, 'routes', this); } + removeRoute(route) { removeItems(this, 'routes', [route]); } + removeRoutes(routes) { removeItems(this, 'routes', routes); } + setIndexedRoutes(indexedRoutes) { setIndexedItems(this, 'routes', indexedRoutes); } /* trips.txt */ - addTrip(trip) { addItems([trip], 'trips', this); } - addTrips(trips) { addItems(trips, 'trips', this); } - forEachTrip(iterator) { forEachItem(iterator, 'trips', this); } - getIndexedTrips() { return getIndexedTableOfGtfs('trips', this); } + addTrip(trip) { addItems(this, 'trips', [trip]); } + addTrips(trips) { addItems(this, 'trips', trips); } + forEachTrip(iterator) { forEachItem(this, 'trips', iterator); } + getIndexedTrips() { return getIndexedTable(this, 'trips'); } getTripOfStopTime(stopTime) { return getters.getParentItem(stopTime, 'trips', this); } getTripWithId(tripId) { return getters.getItemWithIndex(tripId, 'trips', this); } - removeTrip(trip) { removeItems([trip], 'trips', this); } - removeTrips(trips) { removeItems(trips, 'trips', this); } - setIndexedTrips(indexedTrips) { setIndexedItems(indexedTrips, 'trips', this); } + removeTrip(trip) { removeItems(this, 'trips', [trip]); } + removeTrips(trips) { removeItems(this, 'trips', trips); } + setIndexedTrips(indexedTrips) { setIndexedItems(this, 'trips', indexedTrips); } /* stop_times.txt */ - addStopTime(stopTime) { addItems([stopTime], 'stop_times', this); } - addStopTimes(stopTimes) { addItems(stopTimes, 'stop_times', this); } - forEachStopTime(iterator) { forEachItem(iterator, 'stop_times', this); } + addStopTime(stopTime) { addItems(this, 'stop_times', [stopTime]); } + addStopTimes(stopTimes) { addItems(this, 'stop_times', stopTimes); } + forEachStopTime(iterator) { forEachItem(this, 'stop_times', iterator); } forEachStopTimeOfTrip(trip, iterator) { const stopTimeByStopSequence = this.getStopTimeByStopSequenceOfTrip(trip); if (stopTimeByStopSequence instanceof Map) { stopTimeByStopSequence.forEach(iterator); } } - getIndexedStopTimes() { return getIndexedTableOfGtfs('stop_times', this); } + getIndexedStopTimes() { return getIndexedTable(this, 'stop_times'); } getStopTimeByStopSequenceOfTrip(trip) { return getters.getIndexedItemsWithParent(trip, 'stop_times', this); } getStopTimeWithTripIdAndStopSequence(tripId, stopSequence) { return getters.getItemWithIndexes(tripId, stopSequence, 'stop_times', this); } - removeStopTime(stopTime) { removeItems([stopTime], 'stop_times', this); } - removeStopTimes(stopTimes) { removeItems(stopTimes, 'stop_times', this); } - setIndexedStopTimes(indexedStopTimes) { setIndexedItems(indexedStopTimes, 'stop_times', this); } + removeStopTime(stopTime) { removeItems(this, 'stop_times', [stopTime]); } + removeStopTimes(stopTimes) { removeItems(this, 'stop_times', stopTimes); } + setIndexedStopTimes(indexedStopTimes) { setIndexedItems(this, 'stop_times', indexedStopTimes); } /* calendar.txt */ - addCalendar(calendar) { addItems([calendar], 'calendar', this); } - addCalendars(calendars) { addItems(calendars, 'calendar', this); } - forEachCalendar(iterator) { forEachItem(iterator, 'calendar', this); } + addCalendar(calendar) { addItems(this, 'calendar', [calendar]); } + addCalendars(calendars) { addItems(this, 'calendar', calendars); } + forEachCalendar(iterator) { forEachItem(this, 'calendar', iterator); } getCalendarOfTrip(trip) { return getters.getParentItem(trip, 'calendar', this); } getCalendarOfStopTime(stopTime) { return getters.getGrandParentItem(stopTime, 'trips', 'calendar', this); } getCalendarWithServiceId(serviceId) { return getters.getItemWithIndex(serviceId, 'calendar', this); } - getIndexedCalendars() { return getIndexedTableOfGtfs('calendar', this); } - removeCalendar(calendar) { removeItems([calendar], 'calendar', this); } - removeCalendars(calendars) { removeItems(calendars, 'calendar', this); } - setIndexedCalendars(indexedCalendars) { setIndexedItems(indexedCalendars, 'calendar', this); } + getIndexedCalendars() { return getIndexedTable(this, 'calendar'); } + removeCalendar(calendar) { removeItems(this, 'calendar', [calendar]); } + removeCalendars(calendars) { removeItems(this, 'calendar', calendars); } + setIndexedCalendars(indexedCalendars) { setIndexedItems(this, 'calendar', indexedCalendars); } /* calendar_dates.txt */ - addCalendarDate(calendarDate) { addItems([calendarDate], 'calendar_dates', this); } - addCalendarDates(calendarDates) { addItems(calendarDates, 'calendar_dates', this); } - forEachCalendarDate(iterator) { forEachItem(iterator, 'calendar_dates', this); } + addCalendarDate(calendarDate) { addItems(this, 'calendar_dates', [calendarDate]); } + addCalendarDates(calendarDates) { addItems(this, 'calendar_dates', calendarDates); } + forEachCalendarDate(iterator) { forEachItem(this, 'calendar_dates', iterator); } getCalendarDateByDateOfServiceId(serviceId) { return getters.getIndexedItemsWithParentIndex(serviceId, 'calendar_dates', this); } @@ -301,10 +304,10 @@ class Gtfs { getCalendarDateWithServiceIdAndDate(serviceId, date) { return getters.getItemWithIndexes(serviceId, date, 'calendar_dates', this); } - getIndexedCalendarDates() { return getIndexedTableOfGtfs('calendar_dates', this); } - removeCalendarDate(calendarDate) { removeItems([calendarDate], 'calendar_dates', this); } - removeCalendarDates(calendarDates) { removeItems(calendarDates, 'calendar_dates', this); } - setIndexedCalendarDates(indexedCalendarDates) { setIndexedItems(indexedCalendarDates, 'calendar_dates', this); } + getIndexedCalendarDates() { return getIndexedTable(this, 'calendar_dates'); } + removeCalendarDate(calendarDate) { removeItems(this, 'calendar_dates', [calendarDate]); } + removeCalendarDates(calendarDates) { removeItems(this, 'calendar_dates', calendarDates); } + setIndexedCalendarDates(indexedCalendarDates) { setIndexedItems(this, 'calendar_dates', indexedCalendarDates); } /* fare_attributes.txt */ // Not used, therefore not implemented @@ -313,10 +316,10 @@ class Gtfs { // Not used, therefore not implemented /* shapes.txt */ - addShapePoint(shapePoint) { addItems([shapePoint], 'shapes', this); } - addShapePoints(shapePoints) { addItems(shapePoints, 'shapes', this); } - forEachShapePoint(iterator) { forEachItem(iterator, 'shapes', this); } - getIndexedShapePoints() { return getIndexedTableOfGtfs('shapes', this); } + addShapePoint(shapePoint) { addItems(this, 'shapes', [shapePoint]); } + addShapePoints(shapePoints) { addItems(this, 'shapes', shapePoints); } + forEachShapePoint(iterator) { forEachItem(this, 'shapes', iterator); } + getIndexedShapePoints() { return getIndexedTable(this, 'shapes'); } getShapePointByShapePointSequenceOfShapeId(shapeId) { return getters.getIndexedItemsWithParentIndex(shapeId, 'shapes', this); } @@ -324,37 +327,37 @@ class Gtfs { getShapePointWithTripIdAndShapePointSequence(tripId, shapePointSequence) { return getters.getItemWithIndexes(tripId, shapePointSequence, 'shapes', this); } - removeShapePoint(shapePoint) { removeItems([shapePoint], 'shapes', this); } - removeShapePoints(shapePoints) { removeItems(shapePoints, 'shapes', this); } - setIndexedShapePoints(indexedShapes) { setIndexedItems(indexedShapes, 'shapes', this); } + removeShapePoint(shapePoint) { removeItems(this, 'shapes', [shapePoint]); } + removeShapePoints(shapePoints) { removeItems(this, 'shapes', shapePoints); } + setIndexedShapePoints(indexedShapes) { setIndexedItems(this, 'shapes', indexedShapes); } /* frequencies.txt */ - addFrequency(frequency) { addItems([frequency], 'frequencies', this); } - addFrequencies(frequencies) { addItems(frequencies, 'frequencies', this); } - forEachFrequency(iterator) { forEachItem(iterator, 'frequencies', this); } - getIndexedFrequencies() { return getIndexedTableOfGtfs('frequencies', this); } + addFrequency(frequency) { addItems(this, 'frequencies', [frequency]); } + addFrequencies(frequencies) { addItems(this, 'frequencies', frequencies); } + forEachFrequency(iterator) { forEachItem(this, 'frequencies', iterator); } + getIndexedFrequencies() { return getIndexedTable(this, 'frequencies'); } getFrequencyWithTripIdAndStartTime(tripId, startTime) { return getters.getItemWithIndexes(tripId, startTime, 'frequencies', this); } - removeFrequency(frequency) { removeItems([frequency], 'frequencies', this); } - removeFrequencies(frequencies) { removeItems(frequencies, 'frequencies', this); } - setIndexedFrequencies(indexedFrequencies) { setIndexedItems(indexedFrequencies, 'frequencies', this); } + removeFrequency(frequency) { removeItems(this, 'frequencies', [frequency]); } + removeFrequencies(frequencies) { removeItems(this, 'frequencies', frequencies); } + setIndexedFrequencies(indexedFrequencies) { setIndexedItems(this, 'frequencies', indexedFrequencies); } /* transfers.txt */ - addTransfer(transfer) { addItems([transfer], 'transfers', this); } - addTransfers(transfers) { addItems(transfers, 'transfers', this); } - forEachTransfer(iterator) { forEachItem(iterator, 'transfers', this); } - getIndexedTransfers() { return getIndexedTableOfGtfs('transfers', this); } + addTransfer(transfer) { addItems(this, 'transfers', [transfer]); } + addTransfers(transfers) { addItems(this, 'transfers', transfers); } + forEachTransfer(iterator) { forEachItem(this, 'transfers', iterator); } + getIndexedTransfers() { return getIndexedTable(this, 'transfers'); } getTransfertWithFromStopIdAndToStopId(fromStopId, toStopId) { return getters.getItemWithIndexes(fromStopId, toStopId, 'transfers', this); } - removeTransfer(transfer) { removeItems([transfer], 'transfers', this); } - removeTransfers(transfers) { removeItems(transfers, 'transfers', this); } - setIndexedTransfers(indexedTransfers) { setIndexedItems(indexedTransfers, 'transfers', this); } + removeTransfer(transfer) { removeItems(this, 'transfers', [transfer]); } + removeTransfers(transfers) { removeItems(this, 'transfers', transfers); } + setIndexedTransfers(indexedTransfers) { setIndexedItems(this, 'transfers', indexedTransfers); } /* feed_info.txt */ - getFeedInfo() { return getIndexedTableOfGtfs('feed_info', this); } - setFeedInfo(feedInfo) { setIndexedItems(feedInfo, 'feed_info', this); } + getFeedInfo() { return getIndexedTable(this, 'feed_info'); } + setFeedInfo(feedInfo) { setIndexedItems(this, 'feed_info', feedInfo); } } module.exports = Gtfs; From b307c0d44c0446292fe64fb0bdf78144a9c4ca67 Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Fri, 12 Jan 2018 15:38:08 -0500 Subject: [PATCH 3/9] Add more documention --- gtfs.js | 144 ++++++++++++++++++++++++++++++++++++++++++--- helpers/getters.js | 21 +++++++ 2 files changed, 157 insertions(+), 8 deletions(-) diff --git a/gtfs.js b/gtfs.js index eeb6d16..fd946f7 100644 --- a/gtfs.js +++ b/gtfs.js @@ -18,7 +18,6 @@ const schema = require('./helpers/schema'); * @param {string} tableName Name of the table of the GTFS in which the objects should be added. * @param {Array.} items Array of items to add to the GTFS. */ - function addItems(gtfs, tableName, items) { if (items instanceof Array === false) { throw new Error(`items must be an array instead of: ${items}`); @@ -52,7 +51,6 @@ function addItems(gtfs, tableName, items) { * @param {Object} [options] Configuration object passed to importTable function. * @return {Object} Indexed table returned */ - function getIndexedTable(gtfs, tableName, options) { if (gtfs._tables.has(tableName) === false) { importTable(gtfs, tableName, options); @@ -69,7 +67,6 @@ function getIndexedTable(gtfs, tableName, options) { * @param {string} tableName Name of the table of the GTFS to enumerate. * @param {function} iterator Function which will be applied on every item of the table. */ - function forEachItem(gtfs, tableName, iterator) { if (typeof iterator !== 'function') { throw new Error(`iterator must be a function, instead of a ${typeof iterator}.`); @@ -99,7 +96,6 @@ function forEachItem(gtfs, tableName, iterator) { * @param {string} tableName Name of the table of the GTFS in which to add the items. * @param {Array.} items Array of items to remove of the GTFS. */ - function removeItems(gtfs, tableName, items) { if (items instanceof Array === false) { throw new Error(`items must be an array instead of: ${items}`); @@ -133,7 +129,6 @@ function removeItems(gtfs, tableName, items) { * @param {string} tableName Name of the table of the GTFS to set. * @param {Map} indexedItems Object properly indexed as the schema requires the table to be (see schema.js). */ - function setIndexedItems(gtfs, tableName, indexedItems) { if (indexedItems instanceof Map === false && schema.deepnessByTableName[tableName] !== 0) { throw new Error(`indexedItems must be a Map instead of: ${indexedItems}`); @@ -153,7 +148,6 @@ class Gtfs { * >} [regexPatternObjectsByTableName] Optional ad-hoc regex to fix the tables. See importTable. * @return {Gtfs} gtfs Instanciated GTFS object. */ - constructor(path, regexPatternObjectsByTableName) { if (typeof path !== 'string' || path.length === 0) { throw new Error(`Gtfs need a valid input path as string, instead of: "${path}".`); @@ -172,7 +166,7 @@ class Gtfs { this._tables = new Map(); } - /* io */ + /* Input/Output */ /** * Async function exporting the GTFS at a specific path. * @@ -188,6 +182,7 @@ class Gtfs { */ getPath() { return this._path; } + /* Generic table & item manipulation */ /** * Table-generic function to add an item in a table of the GTFS. @@ -205,25 +200,158 @@ class Gtfs { */ addItemsInTable(items, tableName) { addItems(this, tableName, items); } + /** + * Table-generic function to get an iterate in a table of the GTFS. + * + * @param {string} tableName Name of the table of the GTFS to enumerate. + * @param {function} iterator Function which will be applied on every item of the table. + */ forEachItemInTable(tableName, iterator) { forEachItem(this, tableName, iterator); } + + /** + * Enumerate through every table name of the GTFS. + * + * @param {function} iterator Function which will be applied on every table name. + */ forEachTableName(iterator) { this.getTableNames().forEach(iterator); } - getIndexedTable(tableName, forcedValuesByKeys) { return getIndexedTable(this, tableName, forcedValuesByKeys); } + + /** + * Table-generic function to get an indexed table of the GTFS. The indexation depends of the table, and is defined in + * the schema (see schema.js). + * + * @param {string} tableName Name of the table of the GTFS to get. + * @param {Object} [options] Configuration object passed to importTable function. + * @return {Object} Indexed table returned + */ + getIndexedTable(tableName, options) { return getIndexedTable(this, tableName, options); } + + /** + * Get an item of a table using its index. + * + * WARNING: Will work only for the tables in which such unique indexing value exists (see schema.js for an + * exhaustive list) + * + * @param {string} index Index of the item + * @param {string} tableName Name of the table + */ getItemWithIndexInTable(index, tableName) { return getters.getItemWithIndex(index, tableName, this); } + + /** + * Get a Set containing every table names, either defined as part of the GTFS specification, or already loaded in + * the GTFS. + * + * @return {Set.} Array of the table names + */ getTableNames() { return new Set([...schema.tableNames, ...this._tables.keys()]); } + + /** + * Get the parent item using one of its child. + * + * @param {Object} item The child item. + * @param {string} tableName The name of the table containing the parent item. + * @return {Object} The parent item. + */ getParentItem(item, tableName) { return getters.getParentItem(item, tableName, this); } + + /** + * Table-generic function to remove one item in a table of the GTFS. + * + * @param {Object} item Item to remove of the GTFS. + * @param {string} tableName Name of the table of the GTFS in which to remove the items. + */ removeItemInTable(item, tableName) { removeItems(this, tableName, [item]); } + + /** + * Table-generic function to remove items in a table of the GTFS. + * + * @param {Array.} items Array of items to remove of the GTFS. + * @param {string} tableName Name of the table of the GTFS in which to remove the items. + */ removeItemsInTable(items, tableName) { removeItems(this, tableName, items); } + + /** + * Table-generic function to set an indexed table in the GTFS. + * + * @param {Map} indexedItems Object properly indexed as the schema requires the table to be (see schema.js). + * @param {string} tableName Name of the table of the GTFS to set. + */ setIndexedItemsAsTable(indexedItems, tableName) { setIndexedItems(this, tableName, indexedItems); } + /* agency.txt */ + + /** + * Adds an agency in the GTFS. + * + * @param {Object} agency Agency to add in the GTFS. + */ addAgency(agency) { addItems(this, 'agency', [agency]); } + + /** + * Adds a list of agencies in the GTFS. + * + * @param {Array.} agencies Array of agencies to add in the GTFS. + */ addAgencies(agencies) { addItems(this, 'agency', agencies); } + + /** + * Apply a function to each agency in the GTFS. + * + * @param {function} iterator Function which will be applied on every agency. + */ forEachAgency(iterator) { forEachItem(this, 'agency', iterator); } + + /** + * Get the agency using one of its child route. + * + * WARNING: Will return the agency which is indexed with the route.agency_id of the route passed as argument. If the + * internal value of the agency's agency_id has been changed but not it's indexing, the result will be wrong. + * + * @param {Object} route A route of the GTFS. + * @return {Object} The agency indexed by the route.agency_id of the route passed as parameter. + */ getAgencyOfRoute(route) { return getters.getParentItem(route, 'agency', this); } + + /** + * Get the agency using its agency_id. + * + * WARNING: Will return the agency which is indexed with the agency_id passed as argument. If the internal value of + * the agency's agency_id has been changed but not it's indexing, the result will be wrong. + * + * @param {string} agencyId Index of the agency. + * @return {Object} The agency indexed by the agencyId passed as parameter. + */ getAgencyWithId(agencyId) { return getters.getItemWithIndex(agencyId, 'agency', this); } + + /** + * Get the indexed agencies of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Object} Indexed agencies. + */ getIndexedAgencies() { return getIndexedTable(this, 'agency'); } + + /** + * Removes an agency of the GTFS. + * WARNING: it will remove the agency indexed by the `agency.agency_id` of the agency passed as parameter. + * + * @param {Object} agency Agency to remove of the GTFS. + */ removeAgency(agency) { removeItems(this, 'agency', [agency]); } + + /** + * Removes agencies of the GTFS. + * WARNING: it will remove the agencies indexed by the `agency.agency_id` of the agencies passed as parameter. + * + * @param {Array.} agencies Agencies to remove of the GTFS. + */ removeAgencies(agencies) { removeItems(this, 'agency', agencies); } + + /** + * Set the map of indexed agencies. + * WARNING: the Map should be indexed as defined in schema.js + * + * @param {Map.} indexedAgencies Map of agencies properly indexed (see schema.js). + */ setIndexedAgencies(indexedAgencies) { setIndexedItems(this, 'agency', indexedAgencies); } /* stops.txt */ diff --git a/helpers/getters.js b/helpers/getters.js index 9540b97..a8f2c5b 100644 --- a/helpers/getters.js +++ b/helpers/getters.js @@ -72,6 +72,16 @@ function getIndexedItemsWithParentIndex(parentIndex, tableName, gtfs) { return indexedTable.get(parentIndex); } +/** + * Get the item of a table using its index. + * + * WARNING: Will work only for the tables in which such unique indexing value exists (see schema.js for an + * exhaustive list) + * + * @param {string} index Index of the item + * @param {string} tableName Name of the table + * @param {Gtfs} gtfs Gtfs object + */ function getItemWithIndex(index, tableName, gtfs) { if (schema.deepnessByTableName[tableName] !== 1) { throw new Error(`Cannot access item with only one index in "${tableName}", since the deepness is not 1.`); @@ -85,6 +95,17 @@ function getItemWithIndex(index, tableName, gtfs) { return indexedTable.get(index); } +/** + * Get the item of a table using its indexes. + * + * WARNING: Will work only for the tables which have a double level of indexing, which is required when there is no + * value uniquely identifying each item (see schema.js for an exhaustive list) + * + * @param {string} firstIndex First index of the item + * @param {string} secondIndex Second index of the item + * @param {string} tableName Name of the table + * @param {Gtfs} gtfs Gtfs object + */ function getItemWithIndexes(firstIndex, secondIndex, tableName, gtfs) { if (schema.deepnessByTableName[tableName] !== 2) { throw new Error(`Cannot access item with two indexes in "${tableName}", since the deep is not 2.`); From af8b536fce55b961b4ae890f033e437b540a2b7c Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Mon, 15 Jan 2018 09:00:17 -0500 Subject: [PATCH 4/9] Add documentation of the getters --- helpers/getters.js | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/helpers/getters.js b/helpers/getters.js index a8f2c5b..f230ba3 100644 --- a/helpers/getters.js +++ b/helpers/getters.js @@ -2,6 +2,15 @@ const schema = require('./schema'); +/** + * Get the grand parent item using one of its child. + * + * @param {Object} itemWithForeignIndexId The child item. + * @param {string} parentTableName The name of the table containing the parent item. + * @param {string} grandParentTableName The name of the table containing the grand parent item. + * @param {Gtfs} gtfs The GTFS containing the parent item + * @return {Object} The grand parent item. + */ function getGrandParentItem(itemWithForeignIndexId, parentTableName, grandParentTableName, gtfs) { if ( itemWithForeignIndexId === undefined || @@ -40,6 +49,14 @@ function getGrandParentItem(itemWithForeignIndexId, parentTableName, grandParent return gtfs.getItemWithIndexInTable(parentItem[grandParentIndexKey], grandParentTableName); } +/** + * Get the child items of an item. + * + * @param {Object} parentItem The parent item. + * @param {string} tableName The name of the table containing the child items. + * @param {Gtfs} gtfs The GTFS containing the child items. + * @return {Map.} Indexed child items. + */ function getIndexedItemsWithParent(parentItem, tableName, gtfs) { if (schema.deepnessByTableName[tableName] !== 2) { throw new Error(`Table "${tableName}" is not of deepness 2.`); @@ -59,6 +76,14 @@ function getIndexedItemsWithParent(parentItem, tableName, gtfs) { return indexedTable.get(parentItem[firstIndexKey]); } +/** + * Get the child items of an item using its index. + * + * @param {Object} parentIndex The parent item's index. + * @param {string} tableName The name of the table containing the child items. + * @param {Gtfs} gtfs The GTFS containing the child items. + * @return {Map.} Indexed child items. + */ function getIndexedItemsWithParentIndex(parentIndex, tableName, gtfs) { if (schema.deepnessByTableName[tableName] !== 2) { throw new Error(`Table "${tableName}" is not of deepness 2.`); @@ -122,6 +147,14 @@ function getItemWithIndexes(firstIndex, secondIndex, tableName, gtfs) { return (indexedTable.has(firstIndex)) ? indexedTable.get(firstIndex).get(secondIndex) : null; } +/** + * Get the parent item using one of its child. + * + * @param {Object} itemWithForeignIndexId The child item. + * @param {string} tableName The name of the table containing the parent item. + * @param {Gtfs} gtfs The GTFS containing the parent item + * @return {Object} The parent item. + */ function getParentItem(itemWithForeignIndexId, tableName, gtfs) { if ( itemWithForeignIndexId === undefined || From 4170c8c6108e8878b56bb8c7bebef0ce8bd2646d Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Mon, 15 Jan 2018 10:31:29 -0500 Subject: [PATCH 5/9] Add documentation for all public functions --- gtfs.js | 640 +++++++++++++++++++++++++++++++++++++++++++++- helpers/import.js | 47 +++- tests.js | 12 +- 3 files changed, 675 insertions(+), 24 deletions(-) diff --git a/gtfs.js b/gtfs.js index fd946f7..3e2b735 100644 --- a/gtfs.js +++ b/gtfs.js @@ -49,7 +49,11 @@ function addItems(gtfs, tableName, items) { * @param {Gtfs} gtfs GTFS object containing the table to get. * @param {string} tableName Name of the table of the GTFS to get. * @param {Object} [options] Configuration object passed to importTable function. - * @return {Object} Indexed table returned + * @return { + * Object| + * Map.| + * Map.> + * } Indexed table returned */ function getIndexedTable(gtfs, tableName, options) { if (gtfs._tables.has(tableName) === false) { @@ -127,7 +131,7 @@ function removeItems(gtfs, tableName, items) { * * @param {Gtfs} gtfs GTFS object in which the table will be set. * @param {string} tableName Name of the table of the GTFS to set. - * @param {Map} indexedItems Object properly indexed as the schema requires the table to be (see schema.js). + * @param {Object|Map} indexedItems Object properly indexed as the schema requires the table to be (see schema.js). */ function setIndexedItems(gtfs, tableName, indexedItems) { if (indexedItems instanceof Map === false && schema.deepnessByTableName[tableName] !== 0) { @@ -184,6 +188,7 @@ class Gtfs { /* Generic table & item manipulation */ + /** * Table-generic function to add an item in a table of the GTFS. * @@ -319,28 +324,30 @@ class Gtfs { * the agency's agency_id has been changed but not it's indexing, the result will be wrong. * * @param {string} agencyId Index of the agency. - * @return {Object} The agency indexed by the agencyId passed as parameter. + * @return {Object} The agency indexed by the agency_id passed as parameter. */ getAgencyWithId(agencyId) { return getters.getItemWithIndex(agencyId, 'agency', this); } /** * Get the indexed agencies of the GTFS. The indexation is defined in the schema (see schema.js). * - * @return {Object} Indexed agencies. + * @return {Map.} Indexed agencies. */ getIndexedAgencies() { return getIndexedTable(this, 'agency'); } /** * Removes an agency of the GTFS. - * WARNING: it will remove the agency indexed by the `agency.agency_id` of the agency passed as parameter. + * + * WARNING: It will remove the agency indexed by the `agency.agency_id` of the agency passed as parameter. * * @param {Object} agency Agency to remove of the GTFS. */ removeAgency(agency) { removeItems(this, 'agency', [agency]); } /** - * Removes agencies of the GTFS. - * WARNING: it will remove the agencies indexed by the `agency.agency_id` of the agencies passed as parameter. + * Removes a list of agencies of the GTFS. + * + * WARNING: It will remove the agencies indexed by the `agency.agency_id` of the agencies passed as parameter. * * @param {Array.} agencies Agencies to remove of the GTFS. */ @@ -348,143 +355,756 @@ class Gtfs { /** * Set the map of indexed agencies. - * WARNING: the Map should be indexed as defined in schema.js + * + * WARNING: The Map should be indexed as defined in schema.js * * @param {Map.} indexedAgencies Map of agencies properly indexed (see schema.js). */ setIndexedAgencies(indexedAgencies) { setIndexedItems(this, 'agency', indexedAgencies); } + /* stops.txt */ + + /** + * Adds a stop in the GTFS. + * + * @param {Object} stop Stop to add in the GTFS. + */ addStop(stop) { addItems(this, 'stops', [stop]); } + + /** + * Adds a list of stops in the GTFS. + * + * @param {Array.} stops Array of stops to add in the GTFS. + */ addStops(stops) { addItems(this, 'stops', stops); } + + /** + * Apply a function to each stop in the GTFS. + * + * @param {function} iterator Function which will be applied on every stop. + */ forEachStop(iterator) { forEachItem(this, 'stops', iterator); } + + /** + * Get the indexed stops of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.} Indexed stops. + */ getIndexedStops() { return getIndexedTable(this, 'stops'); } + + /** + * Get the stop using one of its child stopTime. + * + * WARNING: Will return the stop which is indexed with the stopTime.stop_id of the stopTime passed as argument. If the + * internal value of the stop's stop_id has been changed but not it's indexing, the result will be wrong. + * + * @param {Object} stopTime A stopTime of the GTFS. + * @return {Object} The stop indexed by the stopTime.stop_id of the stopTime passed as parameter. + */ getStopOfStopTime(stopTime) { return getters.getParentItem(stopTime, 'stops', this); } + + /** + * Get the stop using its stop_id. + * + * WARNING: Will return the stop which is indexed with the stop_id passed as argument. If the internal value of + * the stop's stop_id has been changed but not it's indexing, the result will be wrong. + * + * @param {string} stopId Index of the stop. + * @return {Object} The stop indexed by the stop_id passed as parameter. + */ getStopWithId(stopId) { return getters.getItemWithIndex(stopId, 'stops', this); } + + /** + * Removes a stop of the GTFS. + * + * WARNING: It will remove the stop indexed by the `stop.stop_id` of the stop passed as parameter. + * + * @param {Object} stop Stop to remove of the GTFS. + */ removeStop(stop) { removeItems(this, 'stops', [stop]); } + + /** + * Removes a list of stops of the GTFS. + * + * WARNING: It will remove the stops indexed by the `stop.stop_id` of the stops passed as parameter. + * + * @param {Array.} stops Stops to remove of the GTFS. + */ removeStops(stops) { removeItems(this, 'stops', stops); } + + /** + * Set the map of indexed stops. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.} indexedStops Map of stops properly indexed (see schema.js). + */ setIndexedStops(indexedStops) { setIndexedItems(this, 'stops', indexedStops); } + /* routes.txt */ + + /** + * Adds a route in the GTFS. + * + * @param {Object} route Route to add in the GTFS. + */ addRoute(route) { addItems(this, 'routes', [route]); } + + /** + * Adds a list of routes in the GTFS. + * + * @param {Array.} routes Array of routes to add in the GTFS. + */ addRoutes(routes) { addItems(this, 'routes', routes); } + + /** + * Apply a function to each route in the GTFS. + * + * @param {function} iterator Function which will be applied on every route. + */ forEachRoute(iterator) { forEachItem(this, 'routes', iterator); } + + /** + * Get the indexed routes of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.} Indexed routes. + */ getIndexedRoutes() { return getIndexedTable(this, 'routes'); } + + /** + * Get the route using one of its grand child stopTime. + * + * @param {Object} stopTime A stopTime of the GTFS. + * @return {Object} The grand parent route of the stopTime. + */ getRouteOfStopTime(stopTime) { return getters.getGrandParentItem(stopTime, 'trips', 'routes', this); } + + /** + * Get the route using one of its child trip. + * + * WARNING: Will return the route which is indexed with the trip.route_id of the trip passed as argument. If the + * internal value of the route's route_id has been changed but not it's indexing, the result will be wrong. + * + * @param {Object} trip A trip of the GTFS. + * @return {Object} The route indexed by the trip.route_id of the trip passed as parameter. + */ getRouteOfTrip(trip) { return getters.getParentItem(trip, 'routes', this); } + + /** + * Get the route using its route_id. + * + * WARNING: Will return the route which is indexed with the route_id passed as argument. If the internal value of + * the route's route_id has been changed but not it's indexing, the result will be wrong. + * + * @param {string} routeId Index of the route. + * @return {Object} The route indexed by the route_id passed as parameter. + */ getRouteWithId(routeId) { return getters.getItemWithIndex(routeId, 'routes', this); } + + /** + * Removes a route of the GTFS. + * + * WARNING: It will remove the route indexed by the `route.route_id` of the route passed as parameter. + * + * @param {Object} route Route to remove of the GTFS. + */ removeRoute(route) { removeItems(this, 'routes', [route]); } + + /** + * Removes a list of routes of the GTFS. + * + * WARNING: It will remove the routes indexed by the `route.route_id` of the routes passed as parameter. + * + * @param {Array.} routes Routes to remove of the GTFS. + */ removeRoutes(routes) { removeItems(this, 'routes', routes); } + + /** + * Set the map of indexed routes. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.} indexedRoutes Map of routes properly indexed (see schema.js). + */ setIndexedRoutes(indexedRoutes) { setIndexedItems(this, 'routes', indexedRoutes); } + /* trips.txt */ + + /** + * Adds a trip in the GTFS. + * + * @param {Object} trip Trip to add in the GTFS. + */ addTrip(trip) { addItems(this, 'trips', [trip]); } + + /** + * Adds a list of trips in the GTFS. + * + * @param {Array.} trips Array of trips to add in the GTFS. + */ addTrips(trips) { addItems(this, 'trips', trips); } + + /** + * Apply a function to each trip in the GTFS. + * + * @param {function} iterator Function which will be applied on every trip. + */ forEachTrip(iterator) { forEachItem(this, 'trips', iterator); } + + /** + * Get the indexed trips of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.} Indexed trips. + */ getIndexedTrips() { return getIndexedTable(this, 'trips'); } + + /** + * Get the trip using one of its child stopTime. + * + * WARNING: Will return the trip which is indexed with the stopTime.trip_id of the stopTime passed as argument. If the + * internal value of the trip's trip_id has been changed but not it's indexing, the result will be wrong. + * + * @param {Object} stopTime A stopTime of the GTFS. + * @return {Object} The trip indexed by the stopTime.trip_id of the stopTime passed as parameter. + */ getTripOfStopTime(stopTime) { return getters.getParentItem(stopTime, 'trips', this); } + + /** + * Get the trip using its trip_id. + * + * WARNING: Will return the trip which is indexed with the trip_id passed as argument. If the internal value of + * the trip's trip_id has been changed but not it's indexing, the result will be wrong. + * + * @param {string} tripId Index of the trip. + * @return {Object} The trip indexed by the trip_id passed as parameter. + */ getTripWithId(tripId) { return getters.getItemWithIndex(tripId, 'trips', this); } + + /** + * Removes a trip of the GTFS. + * + * WARNING: It will remove the trip indexed by the `trip.trip_id` of the trip passed as parameter. + * + * @param {Object} trip Trip to remove of the GTFS. + */ removeTrip(trip) { removeItems(this, 'trips', [trip]); } + + /** + * Removes a list of trips of the GTFS. + * + * WARNING: It will remove the trips indexed by the `trip.trip_id` of the trips passed as parameter. + * + * @param {Array.} trips Trips to remove of the GTFS. + */ removeTrips(trips) { removeItems(this, 'trips', trips); } + + /** + * Set the map of indexed trips. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.} indexedTrips Map of trips properly indexed (see schema.js). + */ setIndexedTrips(indexedTrips) { setIndexedItems(this, 'trips', indexedTrips); } + /* stop_times.txt */ + + /** + * Adds a stopTime in the GTFS. + * + * @param {Object} stopTime StopTime to add in the GTFS. + */ addStopTime(stopTime) { addItems(this, 'stop_times', [stopTime]); } + + /** + * Adds a list of stopTimes in the GTFS. + * + * @param {Array.} stopTimes Array of stopTimes to add in the GTFS. + */ addStopTimes(stopTimes) { addItems(this, 'stop_times', stopTimes); } + + /** + * Apply a function to each stopTime in the GTFS. + * + * @param {function} iterator Function which will be applied on every stopTime. + */ forEachStopTime(iterator) { forEachItem(this, 'stop_times', iterator); } + + /** + * Apply a function to each stopTime of a specific trip in the GTFS. + * + * @param {Object} trip Trip scoping the stopTime in which to enumerate. + * @param {function} iterator Function which will be applied on every stopTime of the trip. + */ forEachStopTimeOfTrip(trip, iterator) { const stopTimeByStopSequence = this.getStopTimeByStopSequenceOfTrip(trip); if (stopTimeByStopSequence instanceof Map) { stopTimeByStopSequence.forEach(iterator); } } + + /** + * Get the indexed stopTimes of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.>} Indexed stopTimes. + */ getIndexedStopTimes() { return getIndexedTable(this, 'stop_times'); } + + /** + * Get the child stopTimes of a trip. + * + * @param {Object} trip The parent trip. + * @return {Map.} Indexed child stopTimes. + */ getStopTimeByStopSequenceOfTrip(trip) { return getters.getIndexedItemsWithParent(trip, 'stop_times', this); } + + /** + * Get a stopTime using its indexes: the tripId and the stopSequence. + * + * @param {string} tripId First index of the stopTime + * @param {string} stopSequence Second index of the stopTime + * @return {Object} StopTime object + */ getStopTimeWithTripIdAndStopSequence(tripId, stopSequence) { return getters.getItemWithIndexes(tripId, stopSequence, 'stop_times', this); } + + /** + * Removes a stopTime of the GTFS. + * + * @param {Object} stopTime StopTime to remove of the GTFS. + */ removeStopTime(stopTime) { removeItems(this, 'stop_times', [stopTime]); } + + /** + * Removes a list of stopTimes of the GTFS. + * + * @param {Array.} stopTimes StopTimes to remove of the GTFS. + */ removeStopTimes(stopTimes) { removeItems(this, 'stop_times', stopTimes); } + + /** + * Set the map of indexed stopTimes. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.>} indexedStopTimes Map of stopTimes properly indexed (see schema.js). + */ setIndexedStopTimes(indexedStopTimes) { setIndexedItems(this, 'stop_times', indexedStopTimes); } + /* calendar.txt */ + + /** + * Adds a calendar in the GTFS. + * + * @param {Object} calendar Calendar to add in the GTFS. + */ addCalendar(calendar) { addItems(this, 'calendar', [calendar]); } + + /** + * Adds a list of calendars in the GTFS. + * + * @param {Array.} calendars Array of calendars to add in the GTFS. + */ addCalendars(calendars) { addItems(this, 'calendar', calendars); } + + /** + * Apply a function to each calendar in the GTFS. + * + * @param {function} iterator Function which will be applied on every calendar. + */ forEachCalendar(iterator) { forEachItem(this, 'calendar', iterator); } + + /** + * Get the calendar using one of its child trip. + * + * WARNING: Will return the calendar which is indexed with the trip.service_id of the trip passed as argument. If the + * internal value of the calendar's service_id has been changed but not it's indexing, the result will be wrong. + * + * @param {Object} trip A trip of the GTFS. + * @return {Object} The calendar indexed by the trip.service_id of the trip passed as parameter. + */ getCalendarOfTrip(trip) { return getters.getParentItem(trip, 'calendar', this); } + + /** + * Get the calendar using one of its grand child stopTime. + * + * @param {Object} stopTime A stopTime of the GTFS. + * @return {Object} The grand parent calendar of the stopTime. + */ getCalendarOfStopTime(stopTime) { return getters.getGrandParentItem(stopTime, 'trips', 'calendar', this); } + + /** + * Get the calendar using its service_id. + * + * WARNING: Will return the calendar which is indexed with the service_id passed as argument. If the internal value of + * the calendar's service_id has been changed but not it's indexing, the result will be wrong. + * + * @param {string} serviceId Index of the calendar. + * @return {Object} The calendar indexed by the service_id passed as parameter. + */ getCalendarWithServiceId(serviceId) { return getters.getItemWithIndex(serviceId, 'calendar', this); } + + /** + * Get the indexed calendars of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.} Indexed calendars. + */ getIndexedCalendars() { return getIndexedTable(this, 'calendar'); } + + /** + * Removes an calendar of the GTFS. + * + * WARNING: It will remove the calendar indexed by the `calendar.service_id` of the calendar passed as parameter. + * + * @param {Object} calendar Calendar to remove of the GTFS. + */ removeCalendar(calendar) { removeItems(this, 'calendar', [calendar]); } + + /** + * Removes a list of calendars of the GTFS. + * + * @param {Array.} calendars Calendars to remove of the GTFS. + */ removeCalendars(calendars) { removeItems(this, 'calendar', calendars); } + + /** + * Set the map of indexed calendars. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.} indexedCalendars Map of calendars properly indexed (see schema.js). + */ setIndexedCalendars(indexedCalendars) { setIndexedItems(this, 'calendar', indexedCalendars); } + /* calendar_dates.txt */ + + /** + * Adds a calendarDate in the GTFS. + * + * @param {Object} calendarDate CalendarDate to add in the GTFS. + */ addCalendarDate(calendarDate) { addItems(this, 'calendar_dates', [calendarDate]); } + + /** + * Adds a list of calendarDates in the GTFS. + * + * @param {Array.} calendarDates Array of calendarDates to add in the GTFS. + */ addCalendarDates(calendarDates) { addItems(this, 'calendar_dates', calendarDates); } + + /** + * Apply a function to each calendarDate in the GTFS. + * + * @param {function} iterator Function which will be applied on every calendarDate. + */ forEachCalendarDate(iterator) { forEachItem(this, 'calendar_dates', iterator); } + + /** + * Get the calendarDates of a serviceId. + * + * @param {string} serviceId The serviceId. + * @return {Map.} Indexed calendarDates. + */ getCalendarDateByDateOfServiceId(serviceId) { return getters.getIndexedItemsWithParentIndex(serviceId, 'calendar_dates', this); } + + /** + * Get the child calendarDates of a trip. + * + * @param {Object} trip The parent trip. + * @return {Map.} Indexed child calendarDates. + */ getCalendarDateByDateOfTrip(trip) { return getters.getIndexedItemsWithParent(trip, 'calendar_dates', this); } + + /** + * Get a calendarDate using its indexes: the serviceId and the date. + * + * @param {string} serviceId First index of the calendarDate + * @param {string} date Second index of the calendarDate + * @return {Object} CalendarDate object + */ getCalendarDateWithServiceIdAndDate(serviceId, date) { return getters.getItemWithIndexes(serviceId, date, 'calendar_dates', this); } + + /** + * Get the indexed calendarDates of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.>} Indexed calendarDates. + */ getIndexedCalendarDates() { return getIndexedTable(this, 'calendar_dates'); } + + /** + * Removes a calendarDate of the GTFS. + * + * @param {Object} calendarDate CalendarDate to remove of the GTFS. + */ removeCalendarDate(calendarDate) { removeItems(this, 'calendar_dates', [calendarDate]); } + + /** + * Removes a list of calendarDates of the GTFS. + * + * @param {Array.} calendarDates CalendarDates to remove of the GTFS. + */ removeCalendarDates(calendarDates) { removeItems(this, 'calendar_dates', calendarDates); } + + /** + * Set the map of indexed calendarDates. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param { + * Map.> + * } indexedCalendarDates Map of calendarDates properly indexed (see schema.js). + */ setIndexedCalendarDates(indexedCalendarDates) { setIndexedItems(this, 'calendar_dates', indexedCalendarDates); } + /* fare_attributes.txt */ // Not used, therefore not implemented + /* fare_rules.txt */ // Not used, therefore not implemented + /* shapes.txt */ + + /** + * Adds a shapePoint in the GTFS. + * + * @param {Object} shapePoint ShapePoint to add in the GTFS. + */ addShapePoint(shapePoint) { addItems(this, 'shapes', [shapePoint]); } + + /** + * Adds a list of shapePoints in the GTFS. + * + * @param {Array.} shapePoints Array of shapePoints to add in the GTFS. + */ addShapePoints(shapePoints) { addItems(this, 'shapes', shapePoints); } + + /** + * Apply a function to each shapePoint in the GTFS. + * + * @param {function} iterator Function which will be applied on every shapePoint. + */ forEachShapePoint(iterator) { forEachItem(this, 'shapes', iterator); } + + /** + * Get the indexed shapePoints of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.>} Indexed shapePoints. + */ getIndexedShapePoints() { return getIndexedTable(this, 'shapes'); } + + /** + * Get the shapePoint of a shapeId. + * + * @param {string} shapeId The shapeId. + * @return {Map.} Indexed shapePoints. + */ getShapePointByShapePointSequenceOfShapeId(shapeId) { return getters.getIndexedItemsWithParentIndex(shapeId, 'shapes', this); } + + /** + * Get the child shapePoints of a trip. + * + * @param {Object} trip The parent trip. + * @return {Map.} Indexed child shapePoints. + */ getShapePointByShapePointSequenceOfTrip(trip) { return getters.getIndexedItemsWithParent(trip, 'shapes', this); } + + /** + * Get a shapePoint using its indexes: the tripId and the shapePointSequence. + * + * @param {string} tripId First index of the shapePoint + * @param {string} shapePointSequence Second index of the shapePoint + * @return {Object} ShapePoint object + */ getShapePointWithTripIdAndShapePointSequence(tripId, shapePointSequence) { return getters.getItemWithIndexes(tripId, shapePointSequence, 'shapes', this); } + + /** + * Removes a shapePoint of the GTFS. + * + * @param {Object} shapePoint ShapePoint to remove of the GTFS. + */ removeShapePoint(shapePoint) { removeItems(this, 'shapes', [shapePoint]); } + + /** + * Removes a list of shapePoints of the GTFS. + * + * @param {Array.} shapePoints ShapePoints to remove of the GTFS. + */ removeShapePoints(shapePoints) { removeItems(this, 'shapes', shapePoints); } - setIndexedShapePoints(indexedShapes) { setIndexedItems(this, 'shapes', indexedShapes); } + + /** + * Set the map of indexed shapePoints. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.} indexedShapePoints Map of shapePoints properly indexed (see schema.js). + */ + setIndexedShapePoints(indexedShapePoints) { setIndexedItems(this, 'shapes', indexedShapePoints); } + /* frequencies.txt */ + + /** + * Adds a frequency in the GTFS. + * + * @param {Object} frequency Frequency to add in the GTFS. + */ addFrequency(frequency) { addItems(this, 'frequencies', [frequency]); } + + /** + * Adds a list of frequencies in the GTFS. + * + * @param {Array.} frequencies Array of frequencies to add in the GTFS. + */ addFrequencies(frequencies) { addItems(this, 'frequencies', frequencies); } + + /** + * Apply a function to each frequency in the GTFS. + * + * @param {function} iterator Function which will be applied on every frequency. + */ forEachFrequency(iterator) { forEachItem(this, 'frequencies', iterator); } + + /** + * Get the indexed frequencies of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.>} Indexed frequencies. + */ getIndexedFrequencies() { return getIndexedTable(this, 'frequencies'); } + + /** + * Get a frequency using its indexes: the tripId and the startTime. + * + * @param {string} tripId First index of the frequency + * @param {string} startTime Second index of the frequency + * @return {Object} Frequency object + */ getFrequencyWithTripIdAndStartTime(tripId, startTime) { return getters.getItemWithIndexes(tripId, startTime, 'frequencies', this); } + + /** + * Removes a frequency of the GTFS. + * + * @param {Object} frequency Frequency to remove of the GTFS. + */ removeFrequency(frequency) { removeItems(this, 'frequencies', [frequency]); } + + /** + * Removes a list of frequencies of the GTFS. + * + * @param {Array.} frequencies Frequencies to remove of the GTFS. + */ removeFrequencies(frequencies) { removeItems(this, 'frequencies', frequencies); } + + /** + * Set the map of indexed frequencies. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.} indexedFrequencies Map of frequencies properly indexed (see schema.js). + */ setIndexedFrequencies(indexedFrequencies) { setIndexedItems(this, 'frequencies', indexedFrequencies); } + /* transfers.txt */ + + /** + * Adds a transfer in the GTFS. + * + * @param {Object} transfer Transfer to add in the GTFS. + */ addTransfer(transfer) { addItems(this, 'transfers', [transfer]); } + + /** + * Adds a list of transfers in the GTFS. + * + * @param {Array.} transfers Array of transfers to add in the GTFS. + */ addTransfers(transfers) { addItems(this, 'transfers', transfers); } + + /** + * Apply a function to each transfer in the GTFS. + * + * @param {function} iterator Function which will be applied on every transfer. + */ forEachTransfer(iterator) { forEachItem(this, 'transfers', iterator); } + + /** + * Get the indexed transfers of the GTFS. The indexation is defined in the schema (see schema.js). + * + * @return {Map.>} Indexed transfers. + */ getIndexedTransfers() { return getIndexedTable(this, 'transfers'); } - getTransfertWithFromStopIdAndToStopId(fromStopId, toStopId) { + + /** + * Get a transfer using its indexes: the fromStopId and the toStopId. + * + * @param {string} fromStopId First index of the transfer + * @param {string} toStopId Second index of the transfer + * @return {Object} Transfer object + */ + getTransferWithFromStopIdAndToStopId(fromStopId, toStopId) { return getters.getItemWithIndexes(fromStopId, toStopId, 'transfers', this); } + + /** + * Removes a transfer of the GTFS. + * + * @param {Object} transfer Transfer to remove of the GTFS. + */ removeTransfer(transfer) { removeItems(this, 'transfers', [transfer]); } + + /** + * Removes a list of transfers of the GTFS. + * + * @param {Array.} transfers Transfers to remove of the GTFS. + */ removeTransfers(transfers) { removeItems(this, 'transfers', transfers); } + + /** + * Set the map of indexed transfers. + * + * WARNING: The Map should be indexed as defined in schema.js + * + * @param {Map.>} indexedTransfers Map of transfers properly indexed (see schema.js). + */ setIndexedTransfers(indexedTransfers) { setIndexedItems(this, 'transfers', indexedTransfers); } + /* feed_info.txt */ + + /** + * Get the feed info of the GTFS, which is unique. + * + * @return {Object} The feed info object. + */ getFeedInfo() { return getIndexedTable(this, 'feed_info'); } + + /** + * Set the feed info of the GTFS, which is unique. + * + * @param {Object} feedInfo The feed info object. + */ setFeedInfo(feedInfo) { setIndexedItems(this, 'feed_info', feedInfo); } } diff --git a/helpers/import.js b/helpers/import.js index 778dcce..7b65970 100644 --- a/helpers/import.js +++ b/helpers/import.js @@ -7,16 +7,47 @@ const fs = require('fs-extra'); const eachWithLog = require('./logging_iterator_wrapper'); const { fromCsvStringToArray } = require('./csv'); -const schema = require('./schema'); -exports.importTable = (gtfs, tableName, options) => { - options = options || {}; - const indexKeys = options.indexKeys || schema.indexKeysByTableName[tableName]; +/** + * + * @param {Gtfs} gtfs The GTFS in which to import the table. + * @param {string} tableName The table of the name to import. + * @param { + * Map.< + * string, + * Array.<{regex: RegExp, pattern: string}> + * > + * } [regexPatternObjectsByTableName] Optional ad-hoc regex to fix the tables. The keys are the tableName like defined + * in schema.js, the value are arrays containing pairs of regex and pattern to be + * applied on the raw table, before parsing. The goal is to fix some bad CSV to make + * them readable. + * + * Example: + * The following raw is invalid according to the CSV specification: + * + * > something,something else,a field "not" properly escaped,one last thing + * + * It could be fixed with: + * { regex: /,a field "not" properly escaped,/g, + * pattern: ',a field ""not"" properly escaped,' } + * + * The regexPatternObjectsByTableName would be: + * + * regexPatternObjectsByTableName = { + * nameOfTheTable: [{ + * regex: /,a field "not" properly escaped,/g, + * pattern: ',a field ""not"" properly escaped,', + * }] + * }; + */ + +exports.importTable = (gtfs, tableName, regexPatternObjectsByTableName) => { + regexPatternObjectsByTableName = regexPatternObjectsByTableName || {}; const fullPath = `${gtfs.getPath() + tableName}.txt`; if (fs.existsSync(fullPath)) { const fileContent = fs.readFileSync(fullPath); - const rows = getRows(fileContent, gtfs._regexPatternObjectsByTableName, tableName); + const rows = getRows(fileContent, regexPatternObjectsByTableName, tableName); gtfs._tables.set(tableName, processRows(gtfs, tableName, indexKeys, rows)); return; @@ -37,14 +68,14 @@ function getRows(buffer, regexPatternObjectsByTableName, tableName) { let position = 0; const batchLength = 50000; let merge; - const regexPatternObjects = regexPatternObjectsByTableName[tableName]; + const regexPatternObjects = regexPatternObjectsByTableName.get(tableName); while (position < buffer.length) { rowsSlice = buffer.toString('utf8', position, Math.min(buffer.length, position + batchLength)); if (regexPatternObjects) { - regexPatternObjects.forEach((regexPatternObject) => { - const modifiedRowsSlice = rowsSlice.replace(regexPatternObject.regex, regexPatternObject.pattern || ''); + regexPatternObjects.forEach(({regex, pattern}) => { + const modifiedRowsSlice = rowsSlice.replace(regex, pattern || ''); if (modifiedRowsSlice !== rowsSlice) { process.notices.addInfo( __filename, `Applying regex replace to table: "${tableName}". regex: "${regexPatternObject.regex}".` diff --git a/tests.js b/tests.js index 51f0cd0..af451d7 100644 --- a/tests.js +++ b/tests.js @@ -533,7 +533,7 @@ describe('Tests on GTFS', () => { expect(sortedKeys(gtfs.getIndexedTransfers())).to.deep.equal(['stop_0', 'stop_1']); - const transfer01 = gtfs.getTransfertWithFromStopIdAndToStopId('stop_0', 'stop_1'); + const transfer01 = gtfs.getTransferWithFromStopIdAndToStopId('stop_0', 'stop_1'); expect(transfer01.transfer_type).to.equal('0'); gtfs.addTransfer({ from_stop_id: 'stop_2', to_stop_id: 'stop_0', transfer_type: '3' }); @@ -545,12 +545,12 @@ describe('Tests on GTFS', () => { ]); expect(sortedKeys(gtfs.getIndexedTransfers())).to.deep.equal(['stop_0', 'stop_1', 'stop_2', 'stop_3', 'stop_4']); - gtfs.removeTransfer(gtfs.getTransfertWithFromStopIdAndToStopId('stop_0', 'stop_1')); + gtfs.removeTransfer(gtfs.getTransferWithFromStopIdAndToStopId('stop_0', 'stop_1')); expect(sortedKeys(gtfs.getIndexedTransfers())).to.deep.equal(['stop_1', 'stop_2', 'stop_3', 'stop_4']); gtfs.removeTransfers([ - gtfs.getTransfertWithFromStopIdAndToStopId('stop_1', 'stop_0'), - gtfs.getTransfertWithFromStopIdAndToStopId('stop_3', 'stop_0'), + gtfs.getTransferWithFromStopIdAndToStopId('stop_1', 'stop_0'), + gtfs.getTransferWithFromStopIdAndToStopId('stop_3', 'stop_0'), ]); expect(sortedKeys(gtfs.getIndexedTransfers())).to.deep.equal(['stop_2', 'stop_4']); @@ -565,9 +565,9 @@ describe('Tests on GTFS', () => { ])], ])); expect(sortedKeys(gtfs.getIndexedTransfers())).to.deep.equal(['stop_0', 'stop_1']); - const transfer02 = gtfs.getTransfertWithFromStopIdAndToStopId('stop_0', 'stop_2'); + const transfer02 = gtfs.getTransferWithFromStopIdAndToStopId('stop_0', 'stop_2'); expect(transfer02.transfer_type).to.equal('3'); - const transfer10 = gtfs.getTransfertWithFromStopIdAndToStopId('stop_1', 'stop_0'); + const transfer10 = gtfs.getTransferWithFromStopIdAndToStopId('stop_1', 'stop_0'); expect(transfer10.transfer_type).to.equal('1'); const transferTypes = []; From eb2fbe4cb12e20bd33c4282455ca70d05a2be4a0 Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Mon, 15 Jan 2018 10:37:12 -0500 Subject: [PATCH 6/9] Fix missing declaration --- helpers/import.js | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/helpers/import.js b/helpers/import.js index 7b65970..19a26f2 100644 --- a/helpers/import.js +++ b/helpers/import.js @@ -7,6 +7,7 @@ const fs = require('fs-extra'); const eachWithLog = require('./logging_iterator_wrapper'); const { fromCsvStringToArray } = require('./csv'); +const schema = require('./schema'); /** * @@ -42,7 +43,8 @@ const { fromCsvStringToArray } = require('./csv'); */ exports.importTable = (gtfs, tableName, regexPatternObjectsByTableName) => { - regexPatternObjectsByTableName = regexPatternObjectsByTableName || {}; + regexPatternObjectsByTableName = regexPatternObjectsByTableName || new Map(); + const indexKeys = schema.indexKeysByTableName[tableName]; const fullPath = `${gtfs.getPath() + tableName}.txt`; if (fs.existsSync(fullPath)) { From f677321810fdd64d526d154c980aa5787b2e9f5a Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Mon, 15 Jan 2018 13:18:57 -0500 Subject: [PATCH 7/9] Add documentation and improve code to apply regex on bad CSV tables --- gtfs.js | 45 +++++++++++++++++----- helpers/csv.js | 4 +- helpers/import.js | 49 ++++++------------------ {sample => samples/1}/agency.txt | 0 {sample => samples/1}/calendar.txt | 0 {sample => samples/1}/calendar_dates.txt | 0 {sample => samples/1}/feed_info.txt | 0 {sample => samples/1}/frequencies.txt | 0 {sample => samples/1}/routes.txt | 0 {sample => samples/1}/shapes.txt | 0 {sample => samples/1}/stop_times.txt | 0 {sample => samples/1}/stops.txt | 0 {sample => samples/1}/transfers.txt | 0 {sample => samples/1}/trips.txt | 0 samples/2/stops.txt | 3 ++ tests.js | 49 +++++++++++++++++------- 16 files changed, 87 insertions(+), 63 deletions(-) rename {sample => samples/1}/agency.txt (100%) rename {sample => samples/1}/calendar.txt (100%) rename {sample => samples/1}/calendar_dates.txt (100%) rename {sample => samples/1}/feed_info.txt (100%) rename {sample => samples/1}/frequencies.txt (100%) rename {sample => samples/1}/routes.txt (100%) rename {sample => samples/1}/shapes.txt (100%) rename {sample => samples/1}/stop_times.txt (100%) rename {sample => samples/1}/stops.txt (100%) rename {sample => samples/1}/transfers.txt (100%) rename {sample => samples/1}/trips.txt (100%) create mode 100644 samples/2/stops.txt diff --git a/gtfs.js b/gtfs.js index 3e2b735..6cb0227 100644 --- a/gtfs.js +++ b/gtfs.js @@ -48,16 +48,15 @@ function addItems(gtfs, tableName, items) { * * @param {Gtfs} gtfs GTFS object containing the table to get. * @param {string} tableName Name of the table of the GTFS to get. - * @param {Object} [options] Configuration object passed to importTable function. * @return { * Object| * Map.| * Map.> * } Indexed table returned */ -function getIndexedTable(gtfs, tableName, options) { +function getIndexedTable(gtfs, tableName) { if (gtfs._tables.has(tableName) === false) { - importTable(gtfs, tableName, options); + importTable(gtfs, tableName); infoLog(`[Importation] Table ${tableName} has been imported.`); } @@ -145,14 +144,39 @@ class Gtfs { /** * Constructor of the GTFS * - * @param {string} path Path to the folder that contains the GTFS text files. - * @param {Map.< - * string, - * Array.<{regex: RegExp, pattern: string}> - * >} [regexPatternObjectsByTableName] Optional ad-hoc regex to fix the tables. See importTable. + * # options.regexPatternObjectsByTableName + * + * Optional ad-hoc list of regex to fix the tables. The keys are the tableName like defined in schema.js, the value + * are arrays containing pairs of regex and pattern to be applied on the raw table, before parsing. The goal is to fix + * some bad CSV to make them readable. + * + * Example + * + * The following raw is invalid according to the CSV specification: + * + * > something,something else,a field "not" properly escaped,one last thing + * + * Assuming it is in someTable.txt, it could be fixed with the following regexPatternObjectsByTableName: + * + * regexPatternObjectsByTableName = { + * nameOfTheTable: [{ + * regex: /,a field "not" properly escaped,/g, + * pattern: ',a field ""not"" properly escaped,', + * }] + * }; + * + * # options.throws + * + * Optional ad-hoc boolean. Default is true. Will force the parser to ignore invalid rows in the tables. + * + * @param {string} path Path to the folder that contains the GTFS text files. + * @param {{ + * regexPatternObjectsByTableName: Map.>, + * throws: boolean + * }} [options] Optional. See list above. * @return {Gtfs} gtfs Instanciated GTFS object. */ - constructor(path, regexPatternObjectsByTableName) { + constructor(path, {regexPatternObjectsByTableName = new Map(), throws = true} = {}) { if (typeof path !== 'string' || path.length === 0) { throw new Error(`Gtfs need a valid input path as string, instead of: "${path}".`); } @@ -166,7 +190,8 @@ class Gtfs { this.isGtfs = true; this._path = path; - this._regexPatternObjectsByTableName = regexPatternObjectsByTableName || {}; + this._regexPatternObjectsByTableName = regexPatternObjectsByTableName; + this._shouldThrow = throws; this._tables = new Map(); } diff --git a/helpers/csv.js b/helpers/csv.js index d9b2552..0ca7967 100644 --- a/helpers/csv.js +++ b/helpers/csv.js @@ -73,7 +73,7 @@ function fromCsvStringToArray(string, tableName) { return fromCsvStringToArray(string, tableName); } process.notices.addWarning(__filename, `Row not valid in table ${tableName}: ${string}`); - return null; + return []; } const a = []; // Initialize array to receive values. @@ -85,7 +85,7 @@ function fromCsvStringToArray(string, tableName) { /* else */ if (m2 !== undefined) a.push(m2.replace(/\\"/g, '"')); else if (m3 !== undefined) a.push(m3); - return ''; // Return empty string. + return []; // Return empty string. }); // Handle special case of empty last value. if (/,\s*$/.test(string)) { diff --git a/helpers/import.js b/helpers/import.js index 19a26f2..6e5efa8 100644 --- a/helpers/import.js +++ b/helpers/import.js @@ -10,48 +10,21 @@ const { fromCsvStringToArray } = require('./csv'); const schema = require('./schema'); /** + * Import a table in the GTFS. * * @param {Gtfs} gtfs The GTFS in which to import the table. * @param {string} tableName The table of the name to import. - * @param { - * Map.< - * string, - * Array.<{regex: RegExp, pattern: string}> - * > - * } [regexPatternObjectsByTableName] Optional ad-hoc regex to fix the tables. The keys are the tableName like defined - * in schema.js, the value are arrays containing pairs of regex and pattern to be - * applied on the raw table, before parsing. The goal is to fix some bad CSV to make - * them readable. - * - * Example: - * The following raw is invalid according to the CSV specification: - * - * > something,something else,a field "not" properly escaped,one last thing - * - * It could be fixed with: - * { regex: /,a field "not" properly escaped,/g, - * pattern: ',a field ""not"" properly escaped,' } - * - * The regexPatternObjectsByTableName would be: - * - * regexPatternObjectsByTableName = { - * nameOfTheTable: [{ - * regex: /,a field "not" properly escaped,/g, - * pattern: ',a field ""not"" properly escaped,', - * }] - * }; */ -exports.importTable = (gtfs, tableName, regexPatternObjectsByTableName) => { - regexPatternObjectsByTableName = regexPatternObjectsByTableName || new Map(); +exports.importTable = (gtfs, tableName) => { const indexKeys = schema.indexKeysByTableName[tableName]; const fullPath = `${gtfs.getPath() + tableName}.txt`; if (fs.existsSync(fullPath)) { const fileContent = fs.readFileSync(fullPath); - const rows = getRows(fileContent, regexPatternObjectsByTableName, tableName); + const rows = getRows(fileContent, gtfs._regexPatternObjectsByTableName.get(tableName), tableName); - gtfs._tables.set(tableName, processRows(gtfs, tableName, indexKeys, rows)); + gtfs._tables.set(tableName, processRows(gtfs, tableName, indexKeys, rows, gtfs._shouldThrow)); return; } @@ -64,13 +37,12 @@ exports.importTable = (gtfs, tableName, regexPatternObjectsByTableName) => { * Private functions */ -function getRows(buffer, regexPatternObjectsByTableName, tableName) { +function getRows(buffer, regexPatternObjects, tableName) { const rows = []; let rowsSlice; let position = 0; const batchLength = 50000; let merge; - const regexPatternObjects = regexPatternObjectsByTableName.get(tableName); while (position < buffer.length) { rowsSlice = buffer.toString('utf8', position, Math.min(buffer.length, position + batchLength)); @@ -78,10 +50,9 @@ function getRows(buffer, regexPatternObjectsByTableName, tableName) { if (regexPatternObjects) { regexPatternObjects.forEach(({regex, pattern}) => { const modifiedRowsSlice = rowsSlice.replace(regex, pattern || ''); + if (modifiedRowsSlice !== rowsSlice) { - process.notices.addInfo( - __filename, `Applying regex replace to table: "${tableName}". regex: "${regexPatternObject.regex}".` - ); + process.notices.addInfo(__filename, `Applying regex replace to table: "${tableName}". regex: "${regex}".`); rowsSlice = modifiedRowsSlice; } }); @@ -102,7 +73,7 @@ function getRows(buffer, regexPatternObjectsByTableName, tableName) { return rows; } -function processRows(gtfs, tableName, indexKeys, rows) { +function processRows(gtfs, tableName, indexKeys, rows, shouldThrow) { let table = new Map(); if (rows === undefined || rows === null || rows.length === 0) { @@ -124,6 +95,10 @@ function processRows(gtfs, tableName, indexKeys, rows) { }, {}); if (sortedKeys.length !== arrayOfValues.length) { + if (shouldThrow === true) { + throw new Error(`Invalid raw in table ${tableName}: ${JSON.stringify(item)}`); + } + process.notices.addWarning(__filename, `Row not valid in table: ${JSON.stringify(item)}`); return; } diff --git a/sample/agency.txt b/samples/1/agency.txt similarity index 100% rename from sample/agency.txt rename to samples/1/agency.txt diff --git a/sample/calendar.txt b/samples/1/calendar.txt similarity index 100% rename from sample/calendar.txt rename to samples/1/calendar.txt diff --git a/sample/calendar_dates.txt b/samples/1/calendar_dates.txt similarity index 100% rename from sample/calendar_dates.txt rename to samples/1/calendar_dates.txt diff --git a/sample/feed_info.txt b/samples/1/feed_info.txt similarity index 100% rename from sample/feed_info.txt rename to samples/1/feed_info.txt diff --git a/sample/frequencies.txt b/samples/1/frequencies.txt similarity index 100% rename from sample/frequencies.txt rename to samples/1/frequencies.txt diff --git a/sample/routes.txt b/samples/1/routes.txt similarity index 100% rename from sample/routes.txt rename to samples/1/routes.txt diff --git a/sample/shapes.txt b/samples/1/shapes.txt similarity index 100% rename from sample/shapes.txt rename to samples/1/shapes.txt diff --git a/sample/stop_times.txt b/samples/1/stop_times.txt similarity index 100% rename from sample/stop_times.txt rename to samples/1/stop_times.txt diff --git a/sample/stops.txt b/samples/1/stops.txt similarity index 100% rename from sample/stops.txt rename to samples/1/stops.txt diff --git a/sample/transfers.txt b/samples/1/transfers.txt similarity index 100% rename from sample/transfers.txt rename to samples/1/transfers.txt diff --git a/sample/trips.txt b/samples/1/trips.txt similarity index 100% rename from sample/trips.txt rename to samples/1/trips.txt diff --git a/samples/2/stops.txt b/samples/2/stops.txt new file mode 100644 index 0000000..4f966af --- /dev/null +++ b/samples/2/stops.txt @@ -0,0 +1,3 @@ +stop_id,stop_code,stop_name,stop_desc,stop_lat,stop_lon +stop_0,SC0,Stop 0,Some stop,37.728631,-122.431282 +stop_1,SC1,Stop 1,Some "other" stop,37.74103,-122.422482 diff --git a/tests.js b/tests.js index af451d7..2d09042 100644 --- a/tests.js +++ b/tests.js @@ -12,7 +12,7 @@ const { Gtfs } = require('./index'); describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Test on meta functions', (done) => { - const path = `${__dirname}/sample/`; + const path = `${__dirname}/samples/1/`; const gtfs = new Gtfs(path); expect(gtfs.isGtfs).to.equal(true); @@ -23,7 +23,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Test on generic table functions', (done) => { - const path = `${__dirname}/sample/`; + const path = `${__dirname}/samples/1/`; const gtfs = new Gtfs(path); const indexedAgencies = gtfs.getIndexedTable('agency'); @@ -80,7 +80,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on agencies', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedAgencies())).to.deep.equal(['agency_0']); @@ -121,7 +121,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on stops', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedStops())).to.deep.equal(['stop_0', 'stop_1']); @@ -164,7 +164,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on routes', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedRoutes())).to.deep.equal(['route_0']); @@ -209,7 +209,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on trips', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedTrips())).to.deep.equal(['trip_0']); @@ -250,7 +250,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on stop times', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedStopTimes())).to.deep.equal(['trip_0']); @@ -314,7 +314,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on calendars', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedCalendars())).to.deep.equal(['service_0']); @@ -359,7 +359,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on calendar dates', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedCalendarDates())).to.deep.equal(['service_0']); @@ -420,7 +420,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on shapes', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedShapePoints())).to.deep.equal(['shape_0']); @@ -477,7 +477,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on frequencies', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedFrequencies())).to.deep.equal(['trip_0']); @@ -528,7 +528,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on transfers', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(sortedKeys(gtfs.getIndexedTransfers())).to.deep.equal(['stop_0', 'stop_1']); @@ -581,7 +581,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on feed info', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); expect(gtfs.getFeedInfo().feed_lang).to.equal('en'); @@ -599,7 +599,7 @@ describe('Tests on GTFS', () => { // eslint-disable-next-line no-undef it('Tests on exporting', (done) => { - const path = `${__dirname}/sample`; + const path = `${__dirname}/samples/1`; const gtfs = new Gtfs(path); gtfs.getFeedInfo().feed_lang = 'fr'; @@ -641,6 +641,27 @@ describe('Tests on GTFS', () => { }); }); }); + + // eslint-disable-next-line no-undef + it('Tests on the regex/pattern applied to fix a bad CSV', (done) => { + const path = `${__dirname}/samples/2/`; + const gtfsWithoutFix = new Gtfs(path); + + expect(() => gtfsWithoutFix.getIndexedStops()).to.throw(); + + const gtfsWithoutFixWithoutThrow = new Gtfs(path, { throws: false }); + + expect(() => gtfsWithoutFixWithoutThrow.getIndexedStops()).to.not.throw(); + + const regexPatternObjectsByTableName = new Map([[ + 'stops', [{regex: /,Some "other" stop,/g, pattern: ',"Some ""other"" stop",'}], + ]]); + const gtfsWithFix = new Gtfs(path, { regexPatternObjectsByTableName }); + + expect(gtfsWithFix.getStopWithId('stop_1').stop_desc).to.equal('Some "other" stop'); + + done(); + }); }); function sortedKeys(map) { From 744a1b854e0fbd5fe53924d4496b618e98549bf6 Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Mon, 15 Jan 2018 13:22:11 -0500 Subject: [PATCH 8/9] Remove options for getIndexedTable --- gtfs.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gtfs.js b/gtfs.js index 6cb0227..3c27b5f 100644 --- a/gtfs.js +++ b/gtfs.js @@ -250,10 +250,9 @@ class Gtfs { * the schema (see schema.js). * * @param {string} tableName Name of the table of the GTFS to get. - * @param {Object} [options] Configuration object passed to importTable function. * @return {Object} Indexed table returned */ - getIndexedTable(tableName, options) { return getIndexedTable(this, tableName, options); } + getIndexedTable(tableName) { return getIndexedTable(this, tableName); } /** * Get an item of a table using its index. From 52b2bc2c0865fecaba515f3453c2927ddb305db6 Mon Sep 17 00:00:00 2001 From: LeoFrachet Date: Mon, 15 Jan 2018 13:22:58 -0500 Subject: [PATCH 9/9] Update version number --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 97415fd..194b627 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@transit/gtfs", - "version": "1.0.1", + "version": "2.0.0", "description": "A Node.js librairie for GTFS", "main": "index.js", "scripts": {