Nodejs: convert string to buffer

Question

I'm trying to write a string to a socket (socket is called "response"). Here is the code I have sofar (I'm trying to implement a byte caching proxy...):

var http = require('http');
var sys=require('sys');

var localHash={};

http.createServer(function(request, response) {
    var proxy = http.createClient(80, request.headers['host'])
    var proxy_request = proxy.request(request.method, request.url, request.headers);
    proxy_request.addListener('response', function (proxy_response) {
    proxy_response.addListener('data', function(x) {
        var responseData=x.toString();
        var f=50;
        var toTransmit="";
        var p=0;        

        var N=responseData.length;
        if(N>f){
            p=Math.floor(N/f);

            var hash="";
            var chunk="";
            for(var i=0;i<p;i++){
                chunk=responseData.substr(f*i,f);
                hash=DJBHash(chunk);
                if(localHash[hash]==undefined){
                    localHash[hash]=chunk;
                    toTransmit=toTransmit+chunk;
                }else{
                    sys.puts("***hit"+chunk);
                    toTransmit=toTransmit+chunk;//"***EOH"+hash;
                }
            }
            //remainder:
            chunk=responseData.substr(f*p);
            hash=DJBHash(chunk);
            if(localHash[hash]==undefined){
                localHash[hash]=chunk;
                toTransmit=toTransmit+chunk;
            }else{
                toTransmit=toTransmit+chunk;//"***EOH"+hash;
            }
        }else{
            toTransmit=responseData;
        }
        response.write(new Buffer(toTransmit));   /*error occurs here */
    });
    proxy_response.addListener('end', function() {
        response.end();
    });
    response.writeHead(proxy_response.statusCode, proxy_response.headers);
    });
    request.addListener('data', function(chunk) {
        sys.puts(chunk);
        proxy_request.write(chunk, 'binary');
    });
    request.addListener('end', function() {
        proxy_request.end();
    });
}).listen(8080);



function DJBHash(str) {
    var hash = 5381;
    for(var i = 0; i < str.length; i++) {
        hash = (((hash << 5) + hash) + str.charCodeAt(i)) & 0xffffffff;
    }
    if(hash<-1){
        hash=hash*-1;
    }
    return hash;
}

The trouble is, I keep getting a "content encoding error" in Firefox. It's as if the gizipped content isn't being transmitted properly. I've ensured that "toTransmit" is the same as "x" via console.log(x) and console.log(toTransmit).

It's worth noting that if I replace response.write(new Buffer(toTransmit)) with simply response.write(x), the proxy works as expected, but I need to do some payload analysis and then pass "toTransmit", not "x".

I've also tried to response.write(toTransmit) (i.e. without the conversion to buffer) and I keep getting the same content encoding error.

I'm really stuck. I thought I had this problem fixed by converting the string to a buffer as per another thread (http://stackoverflow.com/questions/7090510/nodejs-content-encoding-error), but I've re-opened a new thread to discuss this new problem I'm experiencing.

I should add that if I open a page via the proxy in Opera, I get gobblydeegook - it's as if the gzipped data gets corrupted.

Any insight greatly appreciated.

Many thanks in advance,

Answer 1

Without digging very deep into your code, it seems to me that you might want to change

var responseData=x.toString();

to

var responseData=x.toString("binary");

and finally

response.write(new Buffer(toTransmit, "binary"));

Answer 2

How about this?

var responseData = Buffer.from(x, 'utf8');

from: Convert string to buffer Node

Answer 3

From the docs:

Pure Javascript is Unicode friendly but not nice to binary data. When dealing with TCP streams or the file system, it's necessary to handle octet streams. Node has several strategies for manipulating, creating, and consuming octet streams.

Raw data is stored in instances of the Buffer class. A Buffer is similar to an array of integers but corresponds to a raw memory allocation outside the V8 heap. A Buffer cannot be resized.

So, don't use strings for handling binary data.

Change proxy_request.write(chunk, 'binary'); to proxy_request.write(chunk);.

Omit var responseData=x.toString();, that's a bad idea.

Instead of doing substr on a string, use slice on a buffer.

Instead of doing + with strings, use the "concat" method from the buffertools.

Answer 4

Actually, now new Buffer() is deprecated sence node.js v10+, so better to use Buffer.from(,) from

 response.write(new Buffer(toTransmit));

do

 response.write(Buffer.from(toTransmit,'binary'));