nodejs using mongoose error: FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

Question

I have a nodejs program simply just copy a field from a collection to another collection. I wrote two of it. one copies field naming(string), another copies ids(array of string). the collection is not large, roughly only 900 forms to be iterated. I can see it runs and saved some of the form, but I don't understand why this error occurs as the program continues running:

FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

Here is the program:

var mongoose = require('mongoose'),
    config = require('../modules/system/node-js/parseConfig'),
    schemas = require('../modules/system/node-js/schemas.js'),
    cde_schemas = require('../modules/cde/node-js/schemas'),
    form_schemas = require('../modules/form/node-js/schemas'),
    mongo_cde = require('../modules/cde/node-js/mongo-cde'),
    async = require('async');

var mongoUrl = config.mongoUri;
var conn = mongoose.createConnection(mongoUrl);
var DataElement = conn.model('DataElement', cde_schemas.dataElementSchema);
var Form = conn.model('Form', form_schemas.formSchema);

var formCounter = 0;

Form.find({
    archived: null
}).exec(function (err, forms) {
    if (err) {
        console.log(err);
        process.exit(0);
    }
    async.eachSeries(forms, function (form, doneOneForm) {
        console.log("begin " + formCounter + " form id: " + form.tinyId);
        var questionCount = 0;
        var areYouDone = function () {
            console.log(questionCount);
            if (questionCount === 0) {
                form.save(function (err) {
                    if (err)
                        process.exit(1);
                    else {
                        console.log('saved form id: ' + form.tinyId);
                        formCounter++;
                        doneOneForm();
                    }
                });
            }
        };
        var formElements = form.formElements;
        var getQuestions = function (formElements) {
            formElements.forEach(function (fe) {
                if (fe.elementType === 'question') {
                    questionCount++;
                    var cdeTinyId = fe.question.cde.tinyId;
                    var version = fe.question.cde.version;
                    DataElement.findOne({tinyId: cdeTinyId, version: version}).exec(function (err, cde) {
                        questionCount--;
                        if (err) {
                            console.log(err);
                            process.exit(0);
                        }
                        console.log('found cde id: ' + cdeTinyId + ' version: ' + version);
                        if (cde && cde.ids) fe.question.cde.ids = cde.ids;
                        //if I run this program with this comment instead of above, there is no error, but error happens on the ids which is array of string.
                        //fe.question.cde.name = cde.naming[0].designation;
                        else {
                            console.log("no CDE with id: " + cdeTinyId)
                        }
                        areYouDone();
                    });
                }
                else {
                    getQuestions(fe.formElements);
                }
            });
        };
        getQuestions(formElements);
        areYouDone();
        form.markModified('formElements');
    }, function doneAllForms() {
        console.log('finished all forms, # form: ' + formCounter);
        process.exit(0);
    });
});

I solved this issue by having stream instead of async, stream.pause() on('data'). after form.save, stream.resume. — Peter Huang

mscdex mscdex · Accepted Answer · 2016-01-28T23:27:16

Without seeing any output from your logging statements, my guess is that you're getting into some kind of infinite recursion. The likely culprit in the code you've shown thus far is the getQuestions(fe.formElements) line.

Either the formElements property refers to itself (or refers to another element in a similar way that creates a circular reference) and possibly the first value is such that fe.elementType !== 'question' so it just keeps calling the function over and over again and none of the forEach()s ever complete.

I suppose a similar thing could also happen if there is no circular references but the link from one set of formElements to the next is long enough to cause problems and causes getQuestions() to be executed at least once for each forEach().

You may want to start with a smaller collection of forms and/or verify that your fe.elementType values and formElements links/references are what they should be.

nodejs using mongoose error: FATAL ERROR: CALL_AND_RETRY_LAST Allocation failed - process out of memory

1 Answers