json.dumps on dictionary with bytes for keys

Question

Trying to convert dictionary object with keys of type bytes to json using json.dumps(). Format of dictionary object is not known beforehand. Have found solution for arrays or dictionaries with byte values when using json.dumps (Convert bytes embedded in list (or dict) to str for use with json.dumps) but have not found one for byte keys.

import json

class BytesDump(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, bytes):
            return obj.decode()
        return json.JSONEncoder.default(self, obj)

foo = {'name': b'bob', 'age': 33, 'attributes': {'hair': b'brown', 'arms': 2}}
bar = {b'name': b'bob', b'age': 33, b'attributes': {b'hair': b'brown', b'arms': 2}}

print(json.dumps(foo, cls=BytesDump)) # this works
print(json.dumps(bar, cls=BytesDump)) # this doesn't work

Output from above

{"name": "bob", "age": 33, "attributes": {"hair": "brown", "arms": 2}}
Traceback (most recent call last):
  File "./test.py", line 15, in <module>
    print(json.dumps(bar, cls=BytesDump))
  File "/usr/local/lib/python3.6/json/__init__.py", line 238, in dumps
    **kw).encode(obj)
  File "/usr/local/lib/python3.6/json/encoder.py", line 199, in encode
    chunks = self.iterencode(o, _one_shot=True)
  File "/usr/local/lib/python3.6/json/encoder.py", line 257, in iterencode
    return _iterencode(o, 0)
TypeError: keys must be a string

Answer 1

you could pre-process the dictionary to convert the keys as strings recursively if they're bytes

import json
# your dump code for values, unmodified
class BytesDump(json.JSONEncoder):
    def default(self, obj):
        if isinstance(obj, bytes):
            return obj.decode()
        return json.JSONEncoder.default(self, obj)

# recursive key as string conversion for byte keys
def keys_string(d):
    rval = {}
    if not isinstance(d, dict):
        if isinstance(d,(tuple,list,set)):
            v = [keys_string(x) for x in d]
            return v
        else:
            return d

    for k,v in d.items():
        if isinstance(k,bytes):
            k = k.decode()
        if isinstance(v,dict):
            v = keys_string(v)
        elif isinstance(v,(tuple,list,set)):
            v = [keys_string(x) for x in v]
        rval[k] = v
    return rval

print(json.dumps(keys_string(bar), cls=BytesDump))

with:

bar = {b'name': b'bob', b'age': 33, b'attributes': {b'hair': b'brown', b'arms': 2},
b'other': [{b'hair': b'brown', b'arms': 2}]}

prints:

{"attributes": {"hair": "brown", "arms": 2}, "age": 33, "name": "bob", "other": [{"hair": "brown", "arms": 2}]}

Answer 2

Looks like you need to use a recursive utility function to traverse the dictionary and convert the keys and values it encounters along the way that are bytes into strings:

import json

def decode_dict(d):
    result = {}
    for key, value in d.items():
        if isinstance(key, bytes):
            key = key.decode()
        if isinstance(value, bytes):
            value = value.decode()
        elif isinstance(value, dict):
            value = decode_dict(value)
        result.update({key: value})
    return result


bar = {b'name': b'bob', b'age': 33, b'attributes': {b'hair': b'brown', b'arms': 2}}
print(json.dumps(decode_dict(bar)))

Output:

{"name": "bob", "age": 33, "attributes": {"hair": "brown", "arms": 2}}