time-to-botec

async.js (4957B)

---

 /**
 * @license Apache-2.0
 *
 * Copyright (c) 2018 The Stdlib Authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 
 'use strict';
 
 // MODULES //
 
 var logger = require( 'debug' );
 var isStringArray = require( '@stdlib/assert/is-string-array' ).primitives;
 var isFunction = require( '@stdlib/assert/is-function' );
 var copy = require( '@stdlib/utils/copy' );
 var readFile = require( './../../read-file' );
 var config = require( './config.json' );
 var delay = require( './delay.js' );
 var clearPending = require( './clear_pending.js' );
 
 
 // VARIABLES //
 
 var debug = logger( 'read-file-list:async' );
 var MAX_RETRIES = config.max_retries;
 var MAX_DELAY = config.max_delay;
 
 
 // MAIN //
 
 /**
 * Asynchronously reads the entire contents of each file in a file list.
 *
 * @param {StringArray} list - list of file paths
 * @param {(Object|string)} [options] - options
 * @param {(string|null)} [options.encoding] - file encoding
 * @param {string} [options.flag] - file status flag
 * @param {Function} clbk - callback to invoke after reading file contents
 * @throws {TypeError} first argument must be a string array
 * @throws {TypeError} callback argument must be a function
 *
 * @example
 * var list = [ __filename ];
 *
 * readFileList( list, onFiles );
 *
 * function onFiles( error, files ) {
 *     if ( error ) {
 *         throw error;
 *     }
 *     console.dir( files );
 * }
 */
 function readFileList( list, options, clbk ) {
 	var pending;
 	var results;
 	var errFLG;
 	var count;
 	var opts;
 	var len;
 	var cb;
 	var i;
 
 	if ( !isStringArray( list ) ) {
 		throw new TypeError( 'invalid argument. First argument must be a string array. Value: `' + list + '`.' );
 	}
 	if ( arguments.length < 3 ) {
 		opts = {};
 		cb = options;
 	} else {
 		opts = copy( options );
 		cb = clbk;
 	}
 	if ( !isFunction( cb ) ) {
 		throw new TypeError( 'invalid argument. Callback argument must be a function. Value: `' + cb + '`.' );
 	}
 	len = list.length;
 
 	results = new Array( len );
 	pending = {};
 	count = 0;
 
 	debug( 'Reading %d files...', len );
 	for ( i = 0; i < len; i++ ) {
 		debug( 'Reading file: %s (%d of %d).', list[ i ], i+1, len );
 		readFile( list[ i ], opts, getCallback( i ) );
 	}
 
 	/**
 	* Returns a callback to be invoked upon reading a file.
 	*
 	* @private
 	* @param {NonNegativeInteger} idx - index
 	* @returns {Callback} callback
 	*/
 	function getCallback( idx ) {
 		var retries;
 		var file;
 		var k;
 
 		file = list[ idx ];
 		k = idx + 1;
 		retries = 0;
 
 		/**
 		* Retries reading a file.
 		*
 		* @private
 		*/
 		function retry() {
 			delete pending[ idx ];
 			debug( 'Reading file: %s (%d of %d). Retry: %d of %d.', file, k, len, retries, MAX_RETRIES );
 			readFile( file, opts, onRead );
 		}
 
 		/**
 		* Callback to be invoked upon reading a file.
 		*
 		* @private
 		* @param {(Error|null)} error - error object
 		* @param {(Buffer|string)} data - file data
 		* @returns {void}
 		*/
 		function onRead( error, data ) {
 			var d;
 			if ( errFLG ) {
 				debug( 'An error has already been returned. Discarding data for file: %s (%d of %d).', file, k, len );
 				return; // prevents `done()` from being called more than once
 			}
 			if ( error ) {
 				debug( 'Encountered an error when reading %s (%d of %d). Error: %s', file, k, len, error.message );
 				if (
 					error.code === 'EMFILE' || // current process
 					error.code === 'ENFILE'    // across entire system
 				) {
 					retries += 1;
 					if ( retries > MAX_RETRIES ) {
 						debug( 'Maximum number of retries exceeded. Too many open files.' );
 						error = new Error( 'max retries exceeded. Too many open files.' );
 						return done( error );
 					}
 					d = delay( retries, MAX_DELAY );
 					debug( 'Too many open files. Will retry reading file %d of %d in %s seconds.', k, len, d/1000 );
 					pending[ idx ] = setTimeout( retry, d );
 					return;
 				}
 				return done( error );
 			}
 			debug( 'Successfully read file: %s (%d of %d).', file, k, len );
 			results[ idx ] = {
 				'file': file,
 				'data': data
 			};
 			count += 1;
 			debug( 'Read %d of %d files.', count, len );
 			if ( count === len ) {
 				return done();
 			}
 		}
 
 		return onRead;
 	}
 
 	/**
 	* Callback invoked upon completion.
 	*
 	* @private
 	* @param {Error} [error] - error object
 	* @returns {void}
 	*/
 	function done( error ) {
 		clearPending( pending );
 		if ( error ) {
 			errFLG = true;
 			return cb( error );
 		}
 		debug( 'Successfully read all files.' );
 		cb( null, results );
 	}
 }
 
 
 // EXPORTS //
 
 module.exports = readFileList;