mongoDB mapreduce taking so long time for running 3m document

Question

I have one collection with 3 million documents. Each document has 40 fields. Fields are like the follow .

{
    "b_date" : "2016-04-05",
    "d_date" : "2016-06-25",
    "pos" : "MISC",
    "origin" : "DXB",
    "destination" : "HGA",
    "pax" : 1,
    "pax_1" : 2
 },   
{
    "b_date" : "2016-04-05",
    "d_date" : "2016-06-25",
    "pos" : "MISC",
    "origin" : "DXB",
    "destination" : "HGA",
    "pax" : 4,
    "pax_1" : 5
 },   
{
    "b_date" : "2016-04-05",
    "d_date" : "2016-06-26",
    "pos" : "MISC",
    "origin" : "DXB",
    "destination" : "HGA",
    "pax" : 3,
    "pax_1" : 3
 }

Now I want to get the sum of pax and pax_1 by grouping b_date,d_date,pos,origin,destination fields. cumulative pax is grouping of pos,origin,destination fields but cumulative pax and pax_1 should increase based on ascending order of b_date and d_date.

Expected result is.

{
    "_id.dep_date" : "2016-04-05",
    "_id.sale_date" : "2016-06-25",
    "_id.pos" : "MISC",
    "_id.origin" : "DXB",
    "_id.destination" : "HGA",
    "value.pax" : 5,
    "value.cumulative_pax":5,
    "value.pax_1" : 7,
    "value.cumulative_pax_1":7,

 },   
{
    "_id.dep_date" : "2016-04-05",
    "_id.sale_date" : "2016-06-26",
    "_id.pos" : "MISC",
    "_id.origin" : "DXB",
    "_id.destination" : "HGA",
    "value.pax" : 3,
    "value.cumulative_pax":8,
    "value.pax_1" : 3,
    "value.cumulative_pax_1":10,
 }

my mapReduce code

db.collection.mapReduce(
function() {
    emit(
    {
    "pos" : this.pos,
    "origin" : this.origin,
    "destination" : this.destination,
    'dep_date': this.d_date,
    'sale_date': this.b_date,

},
    {
        'pax':this.pax,
        'pax_1':this.pax_1,
    }
    );
}
,
function(key,values) {
    paxt = 0;
    paxt_1 = 0;
    for (var i in values){
    paxt += values[i].pax;
    paxt_1 += values[i].pax_1;
    }
    return {'pax':paxt,
    'pax_1':paxt_1,
    };
}
,
{
    'scope':{
        'pos':'',
        'origin':'',
        'destination':'',
        'dep_date': '',
        'sale_date': '',
        'result':{}
    }
    ,
    'finalize':function(key,value) {
        if (pos != key.pos || 
            origin != key.origin || 
            destination != key.destination || 
            ){  
            result['pax'] = 0;
            result['pax_1'] = 0;
            result['cumulative_pax'] = 0;
            result['cumulative_pax_1'] = 0;
        }
            result['pax'] += value.pax;
            result['cumulative_pax'] = value.pax;
            result['pax_1'] += value.pax_1;
            result['cumulative_pax_1'] = value.pax_1;
            pos = key.pos;
            origin = key.origin;
            destination = key.destination;
            dep_date = key.dep_date;
            sale_date = key.sale_date;

        return result;
        }
    ,
    'out':'some_collection'
    }
)

This map reduce returning expected value but it took so much of time like 3 hours. Is that because of 'b_date' and 'd_date' is being string formatted date? or how to do optimization.
Aggregation is returning result within 3 minutes but I couldn't get cumulative pax by using aggregation.

Murugan Perumal Murugan Perumal · Accepted Answer · 2018-03-02T11:05:12

Map Reduce code,

db.collection.mapReduce(
function() {
    emit(
    {
    "pos" : this.pos,
    "origin" : this.origin,
    "destination" : this.destination,
    'dep_date': this.d_date,
    'sale_date': this.b_date,

},
    {
        'pax':this.pax,
        'pax_1':this.pax_1,
    }
    );
}
,
function(key,values) {
    paxt = 0;
    paxt_1 = 0;
    for (var i in values){
    paxt += values[i].pax;
    paxt_1 += values[i].pax_1;
    }
    return {'pax':paxt,
    'pax_1':paxt_1,
    };
}
,
{
    'scope':{
        'pos':'',
        'origin':'',
        'destination':'',
        'dep_date': '',
        'sale_date': '',
        'result':{}
    }
    ,
    'finalize':function(key,value) {
        if (pos != key.pos || 
            origin != key.origin || 
            destination != key.destination || 
            ){  
            result['pax'] = 0;
            result['pax_1'] = 0;
            result['cumulative_pax'] = 0;
            result['cumulative_pax_1'] = 0;
        }
            result['pax'] += value.pax;
            result['cumulative_pax'] = value.pax;
            result['pax_1'] += value.pax_1;
            result['cumulative_pax_1'] = value.pax_1;
            pos = key.pos;
            origin = key.origin;
            destination = key.destination;
            dep_date = key.dep_date;
            sale_date = key.sale_date;

        return result;
        }
    ,
    'out':'some_collection'
    }
)

mongoDB mapreduce taking so long time for running 3m document

1 Answers