Tech Tips

  1. Uncategorized
  2. 89 view

[node.js]Insert restaurant information from TabelogAPI to mongoDB

食べログAPI

Target

Same as title.
It’s just memo to remember how to use config file (yaml) and write log.

Source code

Libraries which following source code use are written below:
  • cheerio : To analyze xml
  • request : To send request and get response from TabelogAPI
  • yaml : To analyze yaml file (config file)
  • fs : To get config file from file system
  • commander : To analyze command line option
  • mongoose : To treat mongoDB
  • log4js : To write log
#!/usr/bin/env node

/**
 * @description
 *      Get restaurant data from tabelog api and insert data to event.
 * @author hyde <zuqqhi2 at gmail.com>
 */

//===============================================
// Load Libraries

var cheerio = require("cheerio");
var request = require("request");
var yaml = require("yaml");
var fs = require("fs");
var command = require("commander");
var mongoose = require("mongoose");
var log4js = require('log4js');

//===============================================
// Logging Setting

log4js.configure({
	appenders: [{
		"type": "dateFile",
		"filename": "./logs/insert_tabelog_datas.log",
		"pattern": "-yyyy-MM-dd"
	}]
});
var logger = log4js.getLogger("dateFile");

//===============================================
// Load config file

var configData = fs.readFileSync("./config/config.yml","utf8");
var config = yaml.eval(configData); 

//===============================================
// Make request URL

var requestBaseUrl = config.tabelog_api.base_url;
var apiKey = config.tabelog_api.api_key;

// paramter
// parse command options and create help command automatically
command
    .version('1.0.0')
    .usage('[option]')
    .option('-p, --prefecture <String>', 'prefecture name (default japan)')
    .option('-n, --pagenum <n>', 'page number (max 60)', parseInt)
    .parse(process.argv);

var pageNumber = 1;
var prefecture = "japan";
if (command.prefecture) prefecture = command.prefecture;
if (command.pagenum)		pageNumber = command.pagenum;

var queryParams = new Array();
queryParams.push("Prefecture=" + prefecture);
queryParams.push("PageNum=" + pageNumber);
queryParams.push("ResultSet=large");
queryParams.push("Key=" + apiKey);

// join
var requestUrl = requestBaseUrl + "?" + queryParams.join("&");

//===============================================
// DB setting

var db = mongoose.connect('mongodb://' + config.event_db.host + '/' + config.event_db.event_data,
	function (err) {
		if (err) {
			logger.error("Connection Fail. mongodb://" + config.event_db.host + "/" + config.event_db.event_data);
		} else {
			logger.info("Connection Success!");
		}
	}
);

var EventsSchema = new mongoose.Schema({
	event_id     : { type: Number, default: 0 },
	genre_id     : Number,
	title        : String,
	image        : String,
	description  : String,
	url          : String,
	station      : String,
	address      : String,
	business_hour: String,
	holiday      : String,
	latitude     : Number,
	longitude    : Number,
	created_time : { type: Date, default: Date.now },
	update_time  : { type: Date, default: Date.now }
});

EventsSchema.pre('save', function(next) {
    if(!this.isNew) return next();

    var model = this;

    model.db.db.executeDbCommand({
        findAndModify: 'current_event_id', // 'コマンド名': '対象のコレクション名'
        query: { name: model.collection.name }, // 検索オプション
        update: { $set: { name: model.collection.name }, $inc: { sequence: 1 } },
        new: true, // 更新したデータを受け取るかどうか
        upsert: true // 見つからなかったら挿入するかどうか
    }, function(err, data) {
        if(!err && data.documents[0].ok) {
            // model.id に取得した値をセット
            model.event_id = data.documents[0].value.sequence;
            next();
        } else {
            next(err || new Error(data.documents[0].errmsg));
        }
    });
});

var Events = db.model("events_tests", EventsSchema);

//===============================================
// Analyze response

logger.info("request url: " + requestUrl);
request({url: requestUrl}, function(error, response, body)
{
	if (!error && response.statusCode == 200) {
		logger.info("response statusCode : " + response.statusCode);

		$ = cheerio.load(body, {ignoreWhitespace: true, xmlMode: true});

		var url = response.request.href;
		var latest_id = 0;
		$("Item").each(function(i, xmlItem) {
			var data = {};
			data["genre_id"] = 1;
			data["title"] = $(xmlItem).children()[1]["children"][0]["data"];
			data["image"] = "";
			data["description"] = "";
			data["url"] = $(xmlItem).children()[2]["children"][0]["data"];
			data["station"] = $(xmlItem).children()[12]["children"][0]["data"];
			data["address"] = $(xmlItem).children()[13]["children"][0]["data"];
			data["business_hour"] = $(xmlItem).children()[15]["children"][0]["data"];
			data["holiday"] = $(xmlItem).children()[16]["children"][0]["data"];
			data["latitude"] = parseFloat($(xmlItem).children()[17]["children"][0]["data"]);
			data["longitude"] = parseFloat($(xmlItem).children()[18]["children"][0]["data"]);

			logger.info("Retrieve data: " + i);
			logger.info(data);

			var newPost = new Events(data);
			newPost.save(function(err) {
				if (err) {
					logger.error("insert error : " + data);
				}
			});

		});
	} else {
		logger.error("response statusCode : " + response.statusCode);
	}
});

tabelog_api:
 api_key: "<This is your application key for TabelogAPI>"
 base_url: "http://api.tabelog.com/Ver2.1/RestaurantSearch/"

event_db:
 host: "localhost"
 event_data: "event_data"

Result

> db.event_tests.find()
{ "event_id" : 5, "genre_id" : 1, "title" : "AFURI", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000005"), "update_time" : ISODate("2013-08-18T08:59:46.757Z"), "created_time" : ISODate("2013-08-18T08:59:46.757Z"), "__v" : 0 }
{ "event_id" : 10, "genre_id" : 1, "title" : "bills 七里ガ浜", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000a"), "update_time" : ISODate("2013-08-18T08:59:46.760Z"), "created_time" : ISODate("2013-08-18T08:59:46.760Z"), "__v" : 0 }
{ "event_id" : 15, "genre_id" : 1, "title" : "銀座天龍", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000f"), "update_time" : ISODate("2013-08-18T08:59:46.762Z"), "created_time" : ISODate("2013-08-18T08:59:46.762Z"), "__v" : 0 }
{ "event_id" : 20, "genre_id" : 1, "title" : "吉村家", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000014"), "update_time" : ISODate("2013-08-18T08:59:46.765Z"), "created_time" : ISODate("2013-08-18T08:59:46.765Z"), "__v" : 0 }
{ "event_id" : 4, "genre_id" : 1, "title" : "うどん 丸香", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000004"), "update_time" : ISODate("2013-08-18T08:59:46.757Z"), "created_time" : ISODate("2013-08-18T08:59:46.757Z"), "__v" : 0 }
{ "event_id" : 9, "genre_id" : 1, "title" : "モンシェール 堂島本店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000009"), "update_time" : ISODate("2013-08-18T08:59:46.759Z"), "created_time" : ISODate("2013-08-18T08:59:46.759Z"), "__v" : 0 }
{ "event_id" : 14, "genre_id" : 1, "title" : "アテスウェイ", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000e"), "update_time" : ISODate("2013-08-18T08:59:46.761Z"), "created_time" : ISODate("2013-08-18T08:59:46.761Z"), "__v" : 0 }
{ "event_id" : 19, "genre_id" : 1, "title" : "麺屋海神", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000013"), "update_time" : ISODate("2013-08-18T08:59:46.764Z"), "created_time" : ISODate("2013-08-18T08:59:46.764Z"), "__v" : 0 }
{ "event_id" : 3, "genre_id" : 1, "title" : "風雲児", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000003"), "update_time" : ISODate("2013-08-18T08:59:46.757Z"), "created_time" : ISODate("2013-08-18T08:59:46.756Z"), "__v" : 0 }
{ "event_id" : 8, "genre_id" : 1, "title" : "あつた蓬莱軒 本店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000008"), "update_time" : ISODate("2013-08-18T08:59:46.759Z"), "created_time" : ISODate("2013-08-18T08:59:46.759Z"), "__v" : 0 }
{ "event_id" : 13, "genre_id" : 1, "title" : "ピッツエリア エ トラットリア ダ イーサ", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000d"), "update_time" : ISODate("2013-08-18T08:59:46.761Z"), "created_time" : ISODate("2013-08-18T08:59:46.761Z"), "__v" : 0 }
{ "event_id" : 18, "genre_id" : 1, "title" : "梅田はがくれ 本店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000012"), "update_time" : ISODate("2013-08-18T08:59:46.763Z"), "created_time" : ISODate("2013-08-18T08:59:46.763Z"), "__v" : 0 }
{ "event_id" : 2, "genre_id" : 1, "title" : "ミート矢澤", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000002"), "update_time" : ISODate("2013-08-18T08:59:46.756Z"), "created_time" : ISODate("2013-08-18T08:59:46.756Z"), "__v" : 0 }
{ "event_id" : 7, "genre_id" : 1, "title" : "イデミ・スギノ", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000007"), "update_time" : ISODate("2013-08-18T08:59:46.758Z"), "created_time" : ISODate("2013-08-18T08:59:46.758Z"), "__v" : 0 }
{ "event_id" : 12, "genre_id" : 1, "title" : "出町ふたば", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000c"), "update_time" : ISODate("2013-08-18T08:59:46.761Z"), "created_time" : ISODate("2013-08-18T08:59:46.761Z"), "__v" : 0 }
{ "event_id" : 17, "genre_id" : 1, "title" : "たいめいけん", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000011"), "update_time" : ISODate("2013-08-18T08:59:46.763Z"), "created_time" : ISODate("2013-08-18T08:59:46.763Z"), "__v" : 0 }
{ "event_id" : 1, "genre_id" : 1, "title" : "六厘舎TOKYO", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000001"), "update_time" : ISODate("2013-08-18T08:59:46.753Z"), "created_time" : ISODate("2013-08-18T08:59:46.752Z"), "__v" : 0 }
{ "event_id" : 6, "genre_id" : 1, "title" : "エッグスンシングス 原宿店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000006"), "update_time" : ISODate("2013-08-18T08:59:46.758Z"), "created_time" : ISODate("2013-08-18T08:59:46.758Z"), "__v" : 0 }
{ "event_id" : 11, "genre_id" : 1, "title" : "うさぎや", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000b"), "update_time" : ISODate("2013-08-18T08:59:46.760Z"), "created_time" : ISODate("2013-08-18T08:59:46.760Z"), "__v" : 0 }
{ "event_id" : 16, "genre_id" : 1, "title" : "つるとんたん 六本木店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000010"), "update_time" : ISODate("2013-08-18T08:59:46.763Z"), "created_time" : ISODate("2013-08-18T08:59:46.763Z"), "__v" : 0 }

Uncategorized recent post

  1. Run Amazon FreeRTOS on M5Stack Core2 for AWS …

  2. Udacity Self-Driving Car Engineer Nanodegree …

  3. Install sbt 1.0.0 and run sample template

  4. Visualization of Neural Network and its Train…

  5. [Machine Learning]Created docker image includ…

関連記事

  1. [LeapMotion][Javascript]Try to…

  2. [AR]Try to use NyARToolkit

  3. Web Server Contribution by Vag…

  4. [Haskell] Make PGM File part3

  5. [php5-fpm]Check point for heav…

  6. [Selenium]Change User Agent wi…

PAGE TOP