Posted in Uncategorized

How do I read a JSON query string?

   February 24, 2021  1 Comment on How do I read a JSON query string?

Dear developers,

I have a problem with express.js JSON string parsing coming from a GET requests.

On the client-side I use jQuery’s datatables which produces valid JSON (I have checked) and sends it to my express.js server with contentType “application/json;charset=utf-8” and http GET method.

Here is the exact client-side code I use:
https://gist.github.com/OliPelz/7d46ac01e888b6fd2829

here is the extracted GET URL from loading the page above:

http://localhost:3000/ajax/phenotypes/withOrg/like/datatables/?[{%%22name%%22:%%22draw%%22,%%22value%%22:1},{%%22name%%22:%%22columns%%22,%%22value%%22:[{%%22data%%22:0,%%22name%%22:%%22%%22,%%22searchable%%22:true,%%22orderable%%22:true,%%22search%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}},{%%22data%%22:1,%%22name%%22:%%22%%22,%%22searchable%%22:true,%%22orderable%%22:true,%%22search%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}}]},{%%22name%%22:%%22order%%22,%%22value%%22:[{%%22column%%22:0,%%22dir%%22:%%22asc%%22}]},{%%22name%%22:%%22start%%22,%%22value%%22:0},{%%22name%%22:%%22length%%22,%%22value%%22:10},{%%22name%%22:%%22search%%22,%%22value%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}}]

as decoded URL for better viewing experience:

http://localhost:3000/ajax/phenotypes/withOrg/like/datatables/?[{"name":"draw","value":1},{"name":"columns","value":[{"data":0,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":1,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}}]},{"name":"order","value":[{"column":0,"dir":"asc"}]},{"name":"start","value":0},{"name":"length","value":10},{"name":"search","value":{"value":"","regex":false}}]

or here the JSON string extracted and prettyprinted with JSLINT (which validated it as “Valid JSON”)
https://gist.github.com/OliPelz/8cf40a2246f843321007

Now on the express.js server-side I use the normal mechanism to parse the received string from the GET request and process it:

 router.get('/withOrg/like/datatables/', function (req, res) {
    console.log("req.query output");
    console.log(req.query);
    var parsedContent = JSON.parse(req.query);
    //... work with parsedContent

The problem here is that the query parameters cannot be decoded, it is no valid JSON and it cannot be parsed, here is the server console output, note the weird json string

GET /STATIC.datatableTest.html 304 30.231 ms - -
req.query output
{ '{"data":0,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":1,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}}': { '{"column":0,"dir":"asc"}': '' } }
GET /ajax/phenotypes/withOrg/like/datatables/?[{%%22name%%22:%%22draw%%22,%%22value%%22:1},{%%22name%%22:%%22columns%%22,%%22value%%22:[{%%22data%%22:0,%%22name%%22:%%22%%22,%%22searchable%%22:true,%%22orderable%%22:true,%%22search%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}},{%%22data%%22:1,%%22name%%22:%%22%%22,%%22searchable%%22:true,%%22orderable%%22:true,%%22search%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}}]},{%%22name%%22:%%22order%%22,%%22value%%22:[{%%22column%%22:0,%%22dir%%22:%%22asc%%22}]},{%%22name%%22:%%22start%%22,%%22value%%22:0},{%%22name%%22:%%22length%%22,%%22value%%22:10},{%%22name%%22:%%22search%%22,%%22value%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}}] 500 1331.073 ms - 1397

and following parse error (which is obious when looking at the received query.output)

Unexpected token u

SyntaxError: Unexpected token u
    at Object.parse (native)
    at module.exports (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/routes/phenotypesAJAX.js:16:19)
    at Layer.handle [as handle_request] (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/layer.js:82:5)
    at next (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/route.js:100:13)
    at Route.dispatch (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/route.js:81:3)
    at Layer.handle [as handle_request] (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/layer.js:82:5)
    at /Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:234:24
    at Function.proto.process_params (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:312:12)
    at /Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:228:12
    at Function.match_layer (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:295:3)

Ive also tried with a “url” approach
the json request string looks different but is also not in right JSON format

 router.get('/withOrg/like/datatables/', function (req, res) {
    //another try
    console.log('url parse output');
    console.log(url.parse(req.url, true).query);
    console.log('output parsed json object using JSON.parse');
    var myObj = JSON.parse(url.parse(req.url, true).query);

different json string output:

GET /STATIC.datatableTest.html 304 27.207 ms - -
url parse output
{ '[{"name":"draw","value":1},{"name":"columns","value":[{"data":0,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":1,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}}]},{"name":"order","value":[{"column":0,"dir":"asc"}]},{"name":"start","value":0},{"name":"length","value":10},{"name":"search","value":{"value":"","regex":false}}]': '' }
output parsed json object using JSON.parse
GET /ajax/phenotypes/withOrg/like/datatables/?[{%%22name%%22:%%22draw%%22,%%22value%%22:1},{%%22name%%22:%%22columns%%22,%%22value%%22:[{%%22data%%22:0,%%22name%%22:%%22%%22,%%22searchable%%22:true,%%22orderable%%22:true,%%22search%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}},{%%22data%%22:1,%%22name%%22:%%22%%22,%%22searchable%%22:true,%%22orderable%%22:true,%%22search%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}}]},{%%22name%%22:%%22order%%22,%%22value%%22:[{%%22column%%22:0,%%22dir%%22:%%22asc%%22}]},{%%22name%%22:%%22start%%22,%%22value%%22:0},{%%22name%%22:%%22length%%22,%%22value%%22:10},{%%22name%%22:%%22search%%22,%%22value%%22:{%%22value%%22:%%22%%22,%%22regex%%22:false}}] 500 765.225 ms - 1397

resulting in the same error

Unexpected token o

SyntaxError: Unexpected token o
    at Object.parse (native)
    at module.exports (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/routes/phenotypesAJAX.js:22:19)
    at Layer.handle [as handle_request] (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/layer.js:82:5)
    at next (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/route.js:100:13)
    at Route.dispatch (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/route.js:81:3)
    at Layer.handle [as handle_request] (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/layer.js:82:5)
    at /Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:234:24
    at Function.proto.process_params (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:312:12)
    at /Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:228:12
    at Function.match_layer (/Users/oliver/Git/TestApp/TestApp-MEAN/genomernai-MEAN/node_modules/express/lib/router/index.js:295:3)

I think the express reading/decoding mechanism of the raw url string has some bugs because I can make it work with the following quick fix by processing the “raw” URL parameters manually

 router.get('/withOrg/like/datatables/', function (req, res) {  
    var params = url.parse(req.url, true).path.split("?")[1];
    console.log(params);
    var params_decoded = decodeURIComponent(params);
    console.log(params_decoded);
    var myObj = JSON.parse(params_decoded);
    console.log("hello " +myObj[0]);
    console.log("name " +myObj[0].name);
    console.log("value " +myObj[0].value);

})

The code above results in a working javascript object parsed from the JSON string which I can access correctly!
Output:

[{"name":"draw","value":1},{"name":"columns","value":[{"data":0,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}},{"data":1,"name":"","searchable":true,"orderable":true,"search":{"value":"","regex":false}}]},{"name":"order","value":[{"column":0,"dir":"asc"}]},{"name":"start","value":0},{"name":"length","value":10},{"name":"search","value":{"value":"","regex":false}}]
hello [object Object]
name draw
value 1

Is this a bug and can be confimed or am I doing something completely stupid here?

I am not behind some proxy, just developing on localhost without any network restrictions.

Thank you for your help

Best,
Oliver

Author: Fantashit

1 thought on “How do I read a JSON query string?”

  1. I think the express reading/decoding mechanism of the raw url string has some bugs because I can make it work with the following quick fix by processing the “raw” URL parameters manually

    Express does not touch the URL in that way. Manually reading req.url vs url.parse has nothing to do with Express. The difference is when you used url.parse you forgot to use decodeURIComponent.

    As for req.query, using that won’t ever work, because it only functions with standard query strings, which are key=value pairs, which the URL you gave was not.

    Here is how you would read that strange URL without Express at all (it will echo back the decoded value or 400:

    var http = require('http')
var url = require('url')

http.createServer(app).listen(4000)

function app(req, res) {
  var query = url.parse(req.url).query
  var obj

  if (query) {
    try {
      obj = JSON.parse(decodeURIComponent(query))
    } catch (e) {
      res.statusCode = 400
      res.end(e.toString())
      return
    }

    res.setHeader('Content-Type', 'application/json')
    res.end(JSON.stringify(obj, null, 2))
    return
  }

  res.statusCode = 400
  res.end('missing query')
}

    And with Express, doing the same as above:

    var express = require('express')
var url = require('url')

var app = express()

app.use(function (req, res, next) {
  var query = url.parse(req.url).query
  var obj

  if (query) {
    obj = JSON.parse(decodeURIComponent(query))
    res.send(obj)
    return
  }

  var error = new Error('missing query')
  error.status = 400
  next(error)
})

app.listen(4000)

Comments are closed.