unable to get all items using query DynamoDB

Question

How to scan all items from AWS dynamodb using node.js. I am posting my code here.

//Load the AWS SDK for Node.js
var AWS = require('aws-sdk');
var unmarshalItem = require('dynamodb-marshaler').unmarshalItem;

// Set the region
AWS.config.update({region: 'us-east-1'});

// Create DynamoDB service object
var b = new AWS.DynamoDB({apiVersion: '2012-08-10'});

exports.handler =  (event, context, callback) => {

var params = {
    TableName: 'IoTdata2',
    FilterExpression: "#deviceid = :unitID and #devicetimestamp BETWEEN :ftimestamp and :ttimestamp",
    ExpressionAttributeNames: {
    "#deviceid": "id",
    "#devicetimestamp": "timestamp"
  },
  ExpressionAttributeValues: {
    ':unitID': {S: 'arena-MXHGMYzBBP5F6jztnLUdCL' },
    ':ftimestamp' : {S: '1584022680000' },
    ':ttimestamp' : {S: '1584023280000' }
   },
};

b.scan(params, onScan);

function onScan(err, data) {
    if (err) {
        console.error("Unable to scan the table. Error JSON:", JSON.stringify(err, null, 2));
    } else {        
        console.log("Scan succeeded.");
        var items = data.Items.map(function(val){ 
            return  unmarshalItem(val); 
        }) 

        // continue scanning if we have more items
        if (typeof data.LastEvaluatedKey != "undefined") {
            console.log("Scanning for more...");
            params.ExclusiveStartKey = data.LastEvaluatedKey;
            b.scan(params, onScan);
        }
    }
    callback(null, items);
}
};

I have followed the link https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/GettingStarted.NodeJs.04.html

I am getting time out here after a while. I have checked this link too How to fetch/scan all items from `AWS dynamodb` using node.js I am unable to return data properly i guess, Any suggestions ? Thanks

An iterative solution to re-running the scan at the last evaluated key would be better than this recursive approach. Accumulate all of the items, and unmarshall them all later would be my suggestion. That's assuming that you genuinely need to read all items into memory at the same time (not ideal if it's a large table). Implement backoff/retry as needed or evaluate your DynamoDB table's ability to support the RCU you need to do this full scan. — jarmod
Look like you are using lambda. Make sure to extend execute time and add async, await to your function. — TonyVo

Aravind Reddy Aravind Reddy · Accepted Answer · 2020-03-12T18:46:01

This worked for me by following Hank solution from here: How to fetch/scan all items from `AWS dynamodb` using node.js

exports.handler = async (event, context, callback) => {
    let params = {
   ExpressionAttributeNames: {
    "#deviceid": "id",
    "#devicetimestamp": "timestamp"
  },
  ExpressionAttributeValues: {
    ':unitID': {S: event.deviceid },
    ':ftimestamp' : {S: event.fromtime },
    ':ttimestamp' : {S: event.totime }
   },
 KeyConditionExpression: '#deviceid = :unitID and #devicetimestamp BETWEEN :ftimestamp and :ttimestamp',
 TableName: 'IoTdata2'
};

    let scanResults = [];
    let items;
    do {
        items = await b.query(params).promise();
        items.Items.map((item) => scanResults.push(unmarshalItem(item)));
        params.ExclusiveStartKey = items.LastEvaluatedKey;
    } while (typeof items.LastEvaluatedKey != "undefined");
    var len = scanResults.length;
    console.log(len);
    callback(null, scanResults);
};

unable to get all items using query DynamoDB

1 Answers