Formatting DynamoDB data to normal JSON in AWS Lambda

Question

I'm using AWS Lambda to scan data from a DynamoDB table. This is what I get in return:

{
  "videos": [
    {
      "file": {
        "S": "file1.mp4"
      },
      "id": {
        "S": "1"
      },
      "canvas": {
        "S": "This is Canvas1"
      }
    },
    {
      "file": {
        "S": "main.mp4"
      },
      "id": {
        "S": "0"
      },
      "canvas": {
        "S": "this is a canvas"
      }
    }
  ]
}

My front-end application is using Ember Data Rest Adapter which does not accepts such response. Is there any way I can get normal JSON format? There is this NPM module called dynamodb-marshaler to convert DynamoDB data to normal JSON. I'm looking for a native solution if possible.

Answer 1

AWS JavaScript SDK was recently updated with Document Client which does exactly what you need. Check the announce and usage examples here: http://blogs.aws.amazon.com/javascript/post/Tx1OVH5LUZAFC6T/Announcing-the-Amazon-DynamoDB-Document-Client-in-the-AWS-SDK-for-JavaScript

Answer 2

Node.js Use the unmarshall function from AWSJavaScriptSDK:

const AWS = require("aws-sdk");
    
exports.handler = function( event, context, callback ) {
    const newImages = event.Records.map(
            (record) => AWS.DynamoDB.Converter.unmarshall(record.dynamodb.NewImage)
    );
    console.log('Converted records', newImages);
    callback(null, `Success`);
}

Python Use TypeDeserializer.deserialize from boto3.dynamodb.types:

import json
from boto3.dynamodb.types import TypeDeserializer

def ddb_deserialize(r, type_deserializer = TypeDeserializer()):
    return type_deserializer.deserialize({"M": r})

def lambda_handler(event, context):
    new_images = [ ddb_deserialize(r["dynamodb"]["NewImage"]) for r in event['Records'] ]
    print('Converted records', json.dumps(new_images, indent=2))

Answer 3

I know is a bit old but I had the same problem processing stream data from dynamoDB in node js lambda function. I used the proposed by @churro

import sdk and output converter

var AWS = require("aws-sdk");
var parse = AWS.DynamoDB.Converter.output;

use the parse function with a small hack

exports.handler = function( event, context, callback ) {
  var docClient = new AWS.DynamoDB.DocumentClient();
  event.Records.forEach((record) => {
        console.log(record.eventID);
        console.log(record.eventName);
        console.log('DynamoDB Record:', parse({ "M": record.dynamodb.NewImage }));
    });
  callback(null, `Successfully processed ${event.Records.length} records.`);
}

Hope it helps

Answer 4

• Javascript: AWS SDK provides the unmarshall function

• Python: use TypeDeserializer from boto3.dynamodb.types:

from boto3.dynamodb.types import TypeDeserializer, TypeSerializer

def from_dynamodb_to_json(item):
    d = TypeDeserializer()
    return {k: d.deserialize(value=v) for k, v in item.items()}

## Usage:
from_dynamodb_to_json({
    "Day": {"S": "Monday"},
    "mylist": {"L": [{"S": "Cookies"}, {"S": "Coffee"}, {"N": "3.14159"}]}
})
# {'Day': 'Monday', 'mylist': ['Cookies', 'Coffee', Decimal('3.14159')]}

Answer 5

Here you can find gist which does that:

function mapper(data) {

let S = "S";
let SS = "SS";
let NN = "NN";
let NS = "NS";
let BS = "BS";
let BB = "BB";
let N = "N";
let BOOL = "BOOL";
let NULL = "NULL";
let M = "M";
let L = "L";

if (isObject(data)) {
    let keys = Object.keys(data);
    while (keys.length) {
        let key = keys.shift();
        let types = data[key];

        if (isObject(types) && types.hasOwnProperty(S)) {
            data[key] = types[S];
        } else if (isObject(types) && types.hasOwnProperty(N)) {
            data[key] = parseFloat(types[N]);
        } else if (isObject(types) && types.hasOwnProperty(BOOL)) {
            data[key] = types[BOOL];
        } else if (isObject(types) && types.hasOwnProperty(NULL)) {
            data[key] = null;
        } else if (isObject(types) && types.hasOwnProperty(M)) {
            data[key] = mapper(types[M]);
        } else if (isObject(types) && types.hasOwnProperty(L)) {
            data[key] = mapper(types[L]);
        } else if (isObject(types) && types.hasOwnProperty(SS)) {
            data[key] = types[SS];
        } else if (isObject(types) && types.hasOwnProperty(NN)) {
            data[key] = types[NN];
        } else if (isObject(types) && types.hasOwnProperty(BB)) {
            data[key] = types[BB];
        } else if (isObject(types) && types.hasOwnProperty(NS)) {
            data[key] = types[NS];
        } else if (isObject(types) && types.hasOwnProperty(BS)) {
            data[key] = types[BS];
        }
    }
}


return data;

function isObject(value) {
    return typeof value === "object" && value !== null;
}

}

https://gist.github.com/igorzg/c80c0de4ad5c4028cb26cfec415cc600

Answer 6

If you are using python in the lambda you can utilise the dynamodb-json library.

Install library

pip install dynamodb-json

and use the below snippet

from dynamodb_json import json_util as util

def marshall(regular_json):
    dynamodb_json = util.dumps(reular_json)

def unmarshall(dynamodb_json):
    regular_json = util.loads(dynamodb_json)

Reference https://pypi.org/project/dynamodb-json/

Answer 7

I think it's just a custom transformation exercise for each app. A simple conversion from DynamoDB's item format to you application format might look like this:

var response = {...} // your response from DynamoDB
var formattedObjects = response.videos.map(function(video) {
    return {
        "file": video.file.S,
        "id": video.id.S,
        "canvas": video.canvas.S
    };
});

If you want to build a generic system for this, you would have to handle DynamoDB's various AttributeValue types. A function like the one below would do the job, but I've left out the hard work of handling most of DynamoDB's more complex attribute value types:

function dynamoItemToPlainObj(dynamoItem) {
    var plainObj = {};
    for (var attributeName in dynamoItem) {
        var attribute = dynamoItem[attributeName];
        var attributeValue;
        for (var itemType in attribute) {
            switch (itemType) {
            case "S":
                attributeValue = attribute.S.toString();
                break;
            case "N":
                attributeValue = Number(attribute.N);
                break;
                // more attribute types...
            default:
                attributeValue = attribute[itemType].toString();
                break;
            }
        }
        plainObj[attributeName] = attributeValue;
    }
    return plainObj;
}    
var formattedObjects = response.videos.map(dynamoItemToPlainObj);

Answer 8

I tried several solutions here but none worked with multi-level data, such as if it includes a list of maps e.g.

{
  "item1": {
    "M": {
      "sub-item1": {
        "L": [
          {
            "M": {
              "sub-item1-list-map": {
                "S": "value"

Below, adapted from @igorzg's answer (which also has that drawback), fixes that.

Example usage:

dynamodb.getItem({...}, function(err, data) {
  if (!err && data && data.Item) {
    var converted = ddb_to_json(data.Item);

Here's the conversion function:

function ddb_to_json(data) {
  
  function isObject(value) {
    return typeof value === "object" && value !== null;
  }

  if(isObject(data))
    return convert_ddb({M:data});

  function convert_ddb(ddbData) {
    if (isObject(ddbData) && ddbData.hasOwnProperty('S'))
      return ddbData.S;
    if (isObject(ddbData) && ddbData.hasOwnProperty('N'))
      return parseFloat(ddbData.N);
    if (isObject(ddbData) && ddbData.hasOwnProperty('BOOL'))
      return ddbData.BOOL;
    if (isObject(ddbData) && ddbData.hasOwnProperty('NULL'))
      return null;
    if (isObject(ddbData) && ddbData.hasOwnProperty('M')) {
      var x = {};
      for(var k in ddbData.M)
        x[k] = convert_ddb(ddbData.M[k])
      return x;
    }
    if (isObject(ddbData) && ddbData.hasOwnProperty('L'))
      return ddbData.L.map(x => convert_ddb(x));
    if (isObject(ddbData) && ddbData.hasOwnProperty('SS'))
      return ddbData.SS;
    if (isObject(ddbData) && ddbData.hasOwnProperty('NN'))
      return ddbData.NN;
    if (isObject(ddbData) && ddbData.hasOwnProperty('BB'))
      return ddbData.BB;
    if (isObject(ddbData) && ddbData.hasOwnProperty('NS'))
      return ddbData.NS;
    if (isObject(ddbData) && ddbData.hasOwnProperty('BS'))
      return ddbData.BS;

    return data;
  }

  return data;
}