Transfer-Encoding chunked and routing seems to cause a problem

[vansen]

Hi everyone,

I am using the http-proxy-middleware inside the node express server. I have setup routing to be done for certain paths, like "api" in this case.
I found everything working fine, but when I encounter a longer response that is being sent from the Backend (API) my Browser receives the following error:
ERR_CONTENT_DECODING_FAILED. I also check if the content passed might be invalid but this is not the case. It is also properly formatted in UTF8.
When I directly address the backend - bypassing the routing I get the correct result, along with the Transfer-Encoding: chunked header. This seems to cause the above problem.
My server.js code looks like this- additionally to the routing I need to check if the url contains "UI" or "API" because I am using a react app where I need to pass on to the index.html file to get it handled. Can anyone please give a hint to where it might be going wrong? Target and routing urls have been "xxx"-ed. The response content is ~500 KB uncompressed, so should not be way too big.

Thanks!

var express = require('express')
var fs = require('fs')
var path = require('path')
var http = require('http');
var proxy = require('http-proxy-middleware')
var winston = require('winston')
var app = express()

const logger = winston.createLogger({
  transports: [
    new winston.transports.Console(),
    new winston.transports.File({ filename: 'log1.log' })
  ]
});

// proxy middleware options
var APICoreOptions = {
    target: 'XXX', // this needs to have a valid value. The target is replaced by the routings defined below.
    changeOrigin: true,               // needed for virtual hosted sites
    ws: true,                         // proxy websockets
    router: {
        'frontend': 'https://backend'
    },
            logProvider: function logProvider(provider) {
                        return winston;
            }, 
            onProxyRes: function OnProxyRes(proxyRes, req, res) {
                        logger.log({
                          level: 'info',
                          message: new Date() + ': onProxyRes. url: ' + res.headers
                        });        
                        
            }, 
            onProxyReq: function OnProxyReq(proxyReq, req, res) {
                        logger.log({
                          level: 'info',
                          message: new Date() + ': onProxyReq. url: ' + req.originalUrl
                        });        
            }, 
            onProxyReqWs: function OnProxyReqWS(proxyReq, req, socket, options, head) {
                        logger.log({
                          level: 'info',
                          message: new Date() + ': onProxyReqWs. url: ' + req.originalUrl
                        });        
            }
            
};

// create the proxy (without context)
var APICoreProxy = proxy(APICoreOptions);

// initiate url rewriting only for '/ui' containint urls
app.use('^/ui/[a-z]*', function (req, res, next) {
            var regexp = new RegExp('/api/');
            
            //res.send('test: ' + res);
            //res.end();
            
            if (regexp.test(req.originalUrl))
            {
                        logger.log({
                          level: 'info',
                          message: 'Passing to API. url: ' + req.originalUrl
                        });        
                        
                        next();
            }
            else
            {

                        logger.log({
                          level: 'info',
                          message: 'Passing to UI. url: ' + req.originalUrl
                        });        
                        
                        res.sendFile(path.join(__dirname, '/dist/index.html'));
            }
            
})

// initiate url rewriting only for '/api' containing urls
app.use('/api', APICoreProxy);

// run server with binaries expected at '/dist' folder
app.use(express.static(__dirname + '/dist'));

//run server
http.createServer(app).listen(process.env.port || 3000);

`

[chimurai]

Hi @vansen

First time I see this issue.

Can you try to reproduce it with an minimal setup;
So with a minimal http-proxy-middleware configuration and minimal express configuration (e.g. without other middlewares)

If problem still persist you can try same minimal setup and replace it with http-proxy.
http-proxy-middleware uses this library to do the actual proxying.

An http-proxy example can be found here:
https://github.com/nodejitsu/node-http-proxy/blob/master/examples/http/standalone-proxy.js#L36

[vansen]

Thanks for your reply, I got it solved at least for now. Further testing is being done currently. I registered to the ProxyRes event and if the destination sends a chunked response, I am currently overwriting that and passing it on "unchunked" by simply removing the transfer-encoding header. This post from nodejitsu got me into the right direction: http-party/node-http-proxy#1007
I will see if I can get the minimal sample setup.
For reference find my changes below:

onProxyRes: function OnProxyRes(proxyRes, req, res) {
        // Capture the response from the backend and check if response is being chunked                             
                               var chunked = /chunked/.test(proxyRes.headers["transfer-encoding"]);
                               if (chunked) {
                                               // If chunked, then gather all chunks and pass it on unchunked. Remove transfer-encoding header.
            proxyRes.headers["transfer-encoding"] = '';
                                               res.write = (function(override) {
                                                               return function(chunk, encoding, callback) {
                                                                              override.call(res, chunk, "binary", callback);
                                                               };
                                               })(res.write);
                                               res.end = (function(override) {
                                                               return function(chunk, encoding, callback) {
                                                                              override.call(res, chunk, "binary", callback);
                                                               };
                                               })(res.end);
                               }

edit: formatting

[helloanoop]

Thanks @vansen This helped me solve my issue.

[nastya-lobova]

Thanks a lot @vansen
My issue has also been solved with your solution.