'Using Promises with fs.readFile in a loop
I'm trying to understand why the below promise setups don't work.
(Note: I already solved this issue with async.map. But I would like to learn why my attempts below didn't work.)
The correct behavior should be: bFunc should run as many time as necessary to fs read all the image files (bFunc below runs twice) and then cFunc console prints "End".
Thanks!
Attempt 1: It runs and stops at cFunc().
var fs = require('fs');
bFunc(0)
.then(function(){ cFunc() }) //cFunc() doesn't run
function bFunc(i){
return new Promise(function(resolve,reject){
var imgPath = __dirname + "/image1" + i + ".png";
fs.readFile(imgPath, function(err, imagebuffer){
if (err) throw err;
console.log(i)
if (i<1) {
i++;
return bFunc(i);
} else {
resolve();
};
});
})
}
function cFunc(){
console.log("End");
}
Attempt 2: In this case, I used a for-loop but it executes out of order. Console prints: End, bFunc done, bFunc done
var fs = require('fs');
bFunc()
.then(function(){ cFunc() })
function bFunc(){
return new Promise(function(resolve,reject){
function read(filepath) {
fs.readFile(filepath, function(err, imagebuffer){
if (err) throw err;
console.log("bFunc done")
});
}
for (var i=0; i<2; i++){
var imgPath = __dirname + "/image1" + i + ".png";
read(imgPath);
};
resolve()
});
}
function cFunc(){
console.log("End");
}
Thanks for the help in advance!
Solution 1:[1]
So, anytime you have multiple async operations to coordinate in some way, I immediately want to go to promises. And, the best way to use promises to coordinate a number of async operations is to make each async operation return a promise. The lowest level async operation you show is fs.readFile()
. Since I use the Bluebird promise library, it has a function for "promisifying" a whole module's worth of async functions.
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
This will create new parallel methods on the fs
object with an "Async" suffix that return promises instead of use straight callbacks. So, there will be an fs.readFileAsync()
that returns a promise. You can read more about Bluebird's promisification here.
So, now you can make a function that gets an image fairly simply and returns a promise whose value is the data from the image:
function getImage(index) {
var imgPath = __dirname + "/image1" + index + ".png";
return fs.readFileAsync(imgPath);
}
Then, in your code, it looks like you want to make bFunc()
be a function that reads three of these images and calls cFunc()
when they are done. You can do that like this:
var Promise = require('bluebird');
var fs = Promise.promisifyAll(require('fs'));
function getImage(index) {
var imgPath = __dirname + "/image1" + index + ".png";
return fs.readFileAsync(imgPath);
}
function getAllImages() {
var promises = [];
// load all images in parallel
for (var i = 0; i <= 2; i++) {
promises.push(getImage(i));
}
// return promise that is resolved when all images are done loading
return Promise.all(promises);
}
getAllImages().then(function(imageArray) {
// you have an array of image data in imageArray
}, function(err) {
// an error occurred
});
If you did not want to use Bluebird, you could manually make a promise version of fs.readFile()
like this:
// make promise version of fs.readFile()
fs.readFileAsync = function(filename) {
return new Promise(function(resolve, reject) {
fs.readFile(filename, function(err, data){
if (err)
reject(err);
else
resolve(data);
});
});
};
Or, in modern versions of node.js, you can use util.promisify()
to make a promisified version of a function that follows the node.js async calling convention:
const util = require('util');
fs.readFileAsync = util.promisify(fs.readFile);
Though, you will quickly find that once you start using promises, you want to use them for all async operations so you'll be "promisifying" lots of things and having a library or at least a generic function that will do that for you will save lots of time.
In even newer versions of node.js (version 10.0+), you can use the built-in version of the fs
library that supports promises:
const fsp = require('fs').promises;
fsp.readFile("someFile").then(data => {
console.log(data);
});
Solution 2:[2]
Node v10 has fs Promises API
const fsPromises = require('fs').promises
const func = async filenames => {
for(let fn of filenames) {
let data = await fsPromises.readFile(fn)
}
}
func(['file1','file2'])
.then(res => console.log('all read', res))
.catch(console.log)
https://nodejs.org/api/fs.html#fs_fs_promises_api
Or if you want to read more files simultaneously:
const func = filenames => {
return Promise.all(
filenames.map(f => fsPromises.readFile(f))
)
}
func(['./a','./b'])
.then(res => console.log('all read', res))
.catch(console.log)
Solution 3:[3]
Your code should look more like this:
// promisify fs.readFile()
fs.readFileAsync = function (filename) {
return new Promise((resolve, reject) => {
fs.readFile(filename, (err, buffer) => {
if (err) reject(err); else resolve(buffer);
});
});
};
const IMG_PATH = "foo";
// utility function
function getImageByIdAsync(i) {
return fs.readFileAsync(IMG_PATH + "/image1" + i + ".png");
}
Usage with a single image:
getImageByIdAsync(0).then(imgBuffer => {
console.log(imgBuffer);
}).catch(err => {
console.error(err);
});
Usage with multiple images:
var images = [1,2,3,4].map(getImageByIdAsync);
Promise.all(images).then(imgBuffers => {
// all images have loaded
}).catch(err => {
console.error(err);
});
To promisify a function means to take an asynchronous function with callback semantics and derive from it a new function with promise semantics.
It can be done manually, like shown above, or – preferably – automatically. Among others, the Bluebird promise library has a helper for that, see http://bluebirdjs.com/docs/api/promisification.html
Solution 4:[4]
If you're using .mjs
ECMAScript Modules import
syntax then here's code that reads a file, based on this GitHub gist comment reply:
import { promises as fs } from 'fs';
let json = await fs.readFile('./package.json', 'utf-8');
console.log(json);
Here's code that does multiple files, based on this answer:
import { promises as fs } from 'fs';
const sequentially = async filenames => {
for(let fn of filenames) {
let json = await fs.readFile(fn, 'utf-8');
console.log(json);
}
}
const parallel = filenames => {
return Promise.all(
filenames.map(fn => fs.readFile(fn, 'utf-8'))
)
}
const fns = ['package.json', 'package-lock.json'];
await sequentially(fns);
await parallel(fns);
console.log('all read');
Solution 5:[5]
you can also use this module: 'fs-readfile-promise'
var readFile = require('fs-readfile-promise');
readFile(__dirname + '/file1.txt','utf-8').then(function (data){
console.log("file's name:", data)
return readFile(__dirname +'/'+data, 'utf-8')
}).then(function (data1){
console.log('Content data:', data1)
}).catch( function (err){
console.log(err)
})
Solution 6:[6]
I once wrote a library for using promises in a loop. It's called for-async
. It's pretty small but getting all the details right is hard, so you can have a look at this for inspiration maybe.
This mainly takes care of running the promise-returning functions in sequence, waiting for the promise to resolve before proceeding to the next item and of handling errors. This differs from Promise.map
because that function runs the promise-returning functions in parallel, which may or may not be what you want.
// example: looping over promise-returning functions with forAsync
forAsync(
['a.txt', 'b.txt', 'c.txt'],
readFile
)
// worker function that returns a promise
function readFile(name, idx) {
return new Promise(function(resolve, reject) {
setTimeout(function(){
console.info('read file ' + idx + ': ' + name)
resolve()
}, 1000)
})
}
// forAsync
function forAsync(arr, work) {
function loop(arr, i) {
return new Promise(function (resolve, reject) {
if (i >= arr.length) {
resolve();
} else try {
Promise.resolve(work(arr[i], i)).then(function () {
return resolve(loop(arr, i + 1));
}).catch(reject);
} catch (error) {
reject(error);
}
});
}
return loop(arr, 0);
}
Sources
This article follows the attribution requirements of Stack Overflow and is licensed under CC BY-SA 3.0.
Source: Stack Overflow
Solution | Source |
---|---|
Solution 1 | |
Solution 2 | |
Solution 3 | |
Solution 4 | Jesus is Lord |
Solution 5 | Changyuan Chen |
Solution 6 | Stijn de Witt |