Tech Tips

Overview

I used to be annoyed by memory leak (and CPU load) when I use node.js for web app development.
I tried to take snapshots and found bottle necks.
This article is how to take snapshot with memory and CPU.
We can check the result of snapshot with Google Chrome browser’s developer tool.

Work Environment

• OS
  • Linux www4322gi 3.2.0-64-generic #97-Ubuntu SMP Wed Jun 4 22:04:21 UTC 2014 x86_64 x86_64 x86_64 GNU/Linux
• Node
  • version: v0.10.26
• Library
  • nodegrind
  • heapdump

Preparation

Install nvm and node.js here.

Bulding

Make template with express

$ vim package.json
{
  "name": "snapshot_test",
  "version": "0.0.1",
  "dependencies": {
    "express": "4.8.7",
    "ejs": "1.0.0",
    "express-generator": "4.8.0"
  }
}

$ npm install

$ ls node_modules/.bin/
express

$ ./node_modules/.bin/express -e -t ejs snapshot_test

$ ls snapshot_test/
app.js  bin  package.json  public  routes  views

Development sample app

$ cd snapshot_test
$ vim package.json
{
  "name": "snapshot_test",
  "version": "0.0.0",
  "private": true,
  "scripts": {
    "start": "node ./bin/www"
  },
  "dependencies": {
    "express": "~4.8.6",
    "body-parser": "~1.6.6",
    "cookie-parser": "~1.3.2",
    "morgan": "~1.2.3",
    "serve-favicon": "~2.0.1",
    "debug": "~1.0.4",
    "ejs": "~0.8.5",
    "request": "2.40.0",
    "cheerio": "0.17.0"
  }
}

$ npm update

routes/index.js

var express = require('express');
var router = express.Router();

var request = require('request');
var cheerio = require('cheerio');

/* GET home page. */
router.get('/', function(req, res) {
  var url = 'http://160.16.71.152:12080/';

  request(url, function (error, response, body) {
    // Connection success
    if (!error && response.statusCode == 200) {
      // Parse
      var $ = cheerio.load(body);
      var hrefs = [];

      // Retrieve all link urls in the page
      $('a').each(function() {
        hrefs.push($(this).attr('href'));
      });

      // return response
      var result = {}
      result.status = 'success';
      result.contents = hrefs;
      res.send(JSON.stringify(result));
    // Error result
    } else {
      var result = {}
      result.status = 'error';
      result.contents = '';
      res.send(JSON.stringify(result));
    }
  })
});

module.exports = router;

Development snapshot part

$ vim package.json
{
  "name": "snapshot_test",
  "version": "0.0.0",
  "private": true,
  "scripts": {
    "start": "node ./bin/www"
  },
  "dependencies": {
    "express": "~4.8.6",
    "body-parser": "~1.6.6",
    "cookie-parser": "~1.3.2",
    "morgan": "~1.2.3",
    "serve-favicon": "~2.0.1",
    "debug": "~1.0.4",
    "ejs": "~0.8.5",
    "request": "2.40.0",
    "cheerio": "0.17.0",
    "nodegrind": "0.1.8",
    "heapdump": "0.2.10"
  }
}

$ npm update

routes/snapcpu.js

var express = require('express');
var router = express.Router();

var fs = require('fs');
var profiler = require('nodegrind');

router.get('/', function(req, res) {
  // filename
  var name =  (new Date().getTime()) + '.cpuprofile';

  // Start to measure
  profiler.startCPU(name);

  // Stop to measure after 5 sec
  setTimeout(function(){
    var cpuProfile = profiler.stopCPU(name, 'cpuprofile');

    //Write profile
    fs.writeFile(name, cpuProfile, function(err){
      if (err) res.send(err);
      else     res.send('OK:' + name);
    });
  }, 5000);
});

module.exports = router;

routes/snapheap.js

var express = require('express');
var router = express.Router();

var fs = require('fs');
var heapdump = require('heapdump');

router.get('/', function(req, res) {
  // filename
  var name = (new Date().getTime()) + '.heapsnapshot';

  // Take a snapshot of heap
  heapdump.writeSnapshot(name, function() {
    res.send('OK:' + name);
  });
});

module.exports = router;

Routing Setting

app.js

var express = require('express');
var path = require('path');
var favicon = require('serve-favicon');
var logger = require('morgan');
var cookieParser = require('cookie-parser');
var bodyParser = require('body-parser');

var routes = require('./routes/index');
var users = require('./routes/users');
var snapcpu = require('./routes/snapcpu');
var snapheap = require('./routes/snapheap');

var app = express();

// view engine setup
app.set('views', path.join(__dirname, 'views'));
app.set('view engine', 'ejs');

// uncomment after placing your favicon in /public
//app.use(favicon(__dirname + '/public/favicon.ico'));
app.use(logger('dev'));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({ extended: false }));
app.use(cookieParser());
app.use(express.static(path.join(__dirname, 'public')));

app.use('/', routes);
app.use('/users', users);
app.use('/snapcpu', snapcpu);
app.use('/snapheap', snapheap);

// catch 404 and forward to error handler
app.use(function(req, res, next) {
    var err = new Error('Not Found');
    err.status = 404;
    next(err);
});

// error handlers

// development error handler
// will print stacktrace
if (app.get('env') === 'development') {
    app.use(function(err, req, res, next) {
        res.status(err.status || 500);
        res.render('error', {
            message: err.message,
            error: err
        });
    });
}

// production error handler
// no stacktraces leaked to user
app.use(function(err, req, res, next) {
    res.status(err.status || 500);
    res.render('error', {
        message: err.message,
        error: {}
    });
});

module.exports = app;

Behavior Check

$ curl http://127.0.0.1:3000
{"status":"success","contents":["http://160.16.71.152:12080/","http://160.16.71.152:12080/?page_id=382",.....................,"http://www.adazing.com/"]}

Check result of snapshot

Take snapshot of heap

$ for i in `seq 5`; do curl 'http://127.0.0.1:3000/'; sleep 1s; done

$ curl http://127.0.0.1:3000/snapheap
OK:1409475822884.heapsnapshot

$ ls
1409475822884.heapsnapshot  app.js  bin  node_modules  package.json  public  routes  views

Take snapshot of CPU

$ for i in `seq 10`; do curl 'http://127.0.0.1:3000'; sleep 1s; done

# Do following command during for loop
$ curl http://127.0.0.1:3000/snapcpu

$ ls
1409475822884.heapsnapshot  1409477190566.cpuprofile  app.js  bin  node_modules  package.json  public  routes  views

Result Check