Categories: Uncategorized

[node.js]Insert restaurant information from TabelogAPI to mongoDB

Target

Same as title.
It’s just memo to remember how to use config file (yaml) and write log.

Source code

Libraries which following source code use are written below:
  • cheerio : To analyze xml
  • request : To send request and get response from TabelogAPI
  • yaml : To analyze yaml file (config file)
  • fs : To get config file from file system
  • commander : To analyze command line option
  • mongoose : To treat mongoDB
  • log4js : To write log
#!/usr/bin/env node

/**
 * @description
 *      Get restaurant data from tabelog api and insert data to event.
 * @author hyde <zuqqhi2 at gmail.com>
 */

//===============================================
// Load Libraries

var cheerio = require("cheerio");
var request = require("request");
var yaml = require("yaml");
var fs = require("fs");
var command = require("commander");
var mongoose = require("mongoose");
var log4js = require('log4js');

//===============================================
// Logging Setting

log4js.configure({
	appenders: [{
		"type": "dateFile",
		"filename": "./logs/insert_tabelog_datas.log",
		"pattern": "-yyyy-MM-dd"
	}]
});
var logger = log4js.getLogger("dateFile");

//===============================================
// Load config file

var configData = fs.readFileSync("./config/config.yml","utf8");
var config = yaml.eval(configData); 

//===============================================
// Make request URL

var requestBaseUrl = config.tabelog_api.base_url;
var apiKey = config.tabelog_api.api_key;

// paramter
// parse command options and create help command automatically
command
    .version('1.0.0')
    .usage('[option]')
    .option('-p, --prefecture <String>', 'prefecture name (default japan)')
    .option('-n, --pagenum <n>', 'page number (max 60)', parseInt)
    .parse(process.argv);

var pageNumber = 1;
var prefecture = "japan";
if (command.prefecture) prefecture = command.prefecture;
if (command.pagenum)		pageNumber = command.pagenum;

var queryParams = new Array();
queryParams.push("Prefecture=" + prefecture);
queryParams.push("PageNum=" + pageNumber);
queryParams.push("ResultSet=large");
queryParams.push("Key=" + apiKey);

// join
var requestUrl = requestBaseUrl + "?" + queryParams.join("&");

//===============================================
// DB setting

var db = mongoose.connect('mongodb://' + config.event_db.host + '/' + config.event_db.event_data,
	function (err) {
		if (err) {
			logger.error("Connection Fail. mongodb://" + config.event_db.host + "/" + config.event_db.event_data);
		} else {
			logger.info("Connection Success!");
		}
	}
);

var EventsSchema = new mongoose.Schema({
	event_id     : { type: Number, default: 0 },
	genre_id     : Number,
	title        : String,
	image        : String,
	description  : String,
	url          : String,
	station      : String,
	address      : String,
	business_hour: String,
	holiday      : String,
	latitude     : Number,
	longitude    : Number,
	created_time : { type: Date, default: Date.now },
	update_time  : { type: Date, default: Date.now }
});

EventsSchema.pre('save', function(next) {
    if(!this.isNew) return next();

    var model = this;

    model.db.db.executeDbCommand({
        findAndModify: 'current_event_id', // 'コマンド名': '対象のコレクション名'
        query: { name: model.collection.name }, // 検索オプション
        update: { $set: { name: model.collection.name }, $inc: { sequence: 1 } },
        new: true, // 更新したデータを受け取るかどうか
        upsert: true // 見つからなかったら挿入するかどうか
    }, function(err, data) {
        if(!err && data.documents[0].ok) {
            // model.id に取得した値をセット
            model.event_id = data.documents[0].value.sequence;
            next();
        } else {
            next(err || new Error(data.documents[0].errmsg));
        }
    });
});

var Events = db.model("events_tests", EventsSchema);

//===============================================
// Analyze response

logger.info("request url: " + requestUrl);
request({url: requestUrl}, function(error, response, body)
{
	if (!error && response.statusCode == 200) {
		logger.info("response statusCode : " + response.statusCode);

		$ = cheerio.load(body, {ignoreWhitespace: true, xmlMode: true});

		var url = response.request.href;
		var latest_id = 0;
		$("Item").each(function(i, xmlItem) {
			var data = {};
			data["genre_id"] = 1;
			data["title"] = $(xmlItem).children()[1]["children"][0]["data"];
			data["image"] = "";
			data["description"] = "";
			data["url"] = $(xmlItem).children()[2]["children"][0]["data"];
			data["station"] = $(xmlItem).children()[12]["children"][0]["data"];
			data["address"] = $(xmlItem).children()[13]["children"][0]["data"];
			data["business_hour"] = $(xmlItem).children()[15]["children"][0]["data"];
			data["holiday"] = $(xmlItem).children()[16]["children"][0]["data"];
			data["latitude"] = parseFloat($(xmlItem).children()[17]["children"][0]["data"]);
			data["longitude"] = parseFloat($(xmlItem).children()[18]["children"][0]["data"]);

			logger.info("Retrieve data: " + i);
			logger.info(data);

			var newPost = new Events(data);
			newPost.save(function(err) {
				if (err) {
					logger.error("insert error : " + data);
				}
			});

		});
	} else {
		logger.error("response statusCode : " + response.statusCode);
	}
});

tabelog_api:
 api_key: "<This is your application key for TabelogAPI>"
 base_url: "http://api.tabelog.com/Ver2.1/RestaurantSearch/"

event_db:
 host: "localhost"
 event_data: "event_data"

Result

> db.event_tests.find()
{ "event_id" : 5, "genre_id" : 1, "title" : "AFURI", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000005"), "update_time" : ISODate("2013-08-18T08:59:46.757Z"), "created_time" : ISODate("2013-08-18T08:59:46.757Z"), "__v" : 0 }
{ "event_id" : 10, "genre_id" : 1, "title" : "bills 七里ガ浜", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000a"), "update_time" : ISODate("2013-08-18T08:59:46.760Z"), "created_time" : ISODate("2013-08-18T08:59:46.760Z"), "__v" : 0 }
{ "event_id" : 15, "genre_id" : 1, "title" : "銀座天龍", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000f"), "update_time" : ISODate("2013-08-18T08:59:46.762Z"), "created_time" : ISODate("2013-08-18T08:59:46.762Z"), "__v" : 0 }
{ "event_id" : 20, "genre_id" : 1, "title" : "吉村家", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000014"), "update_time" : ISODate("2013-08-18T08:59:46.765Z"), "created_time" : ISODate("2013-08-18T08:59:46.765Z"), "__v" : 0 }
{ "event_id" : 4, "genre_id" : 1, "title" : "うどん 丸香", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000004"), "update_time" : ISODate("2013-08-18T08:59:46.757Z"), "created_time" : ISODate("2013-08-18T08:59:46.757Z"), "__v" : 0 }
{ "event_id" : 9, "genre_id" : 1, "title" : "モンシェール 堂島本店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000009"), "update_time" : ISODate("2013-08-18T08:59:46.759Z"), "created_time" : ISODate("2013-08-18T08:59:46.759Z"), "__v" : 0 }
{ "event_id" : 14, "genre_id" : 1, "title" : "アテスウェイ", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000e"), "update_time" : ISODate("2013-08-18T08:59:46.761Z"), "created_time" : ISODate("2013-08-18T08:59:46.761Z"), "__v" : 0 }
{ "event_id" : 19, "genre_id" : 1, "title" : "麺屋海神", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000013"), "update_time" : ISODate("2013-08-18T08:59:46.764Z"), "created_time" : ISODate("2013-08-18T08:59:46.764Z"), "__v" : 0 }
{ "event_id" : 3, "genre_id" : 1, "title" : "風雲児", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000003"), "update_time" : ISODate("2013-08-18T08:59:46.757Z"), "created_time" : ISODate("2013-08-18T08:59:46.756Z"), "__v" : 0 }
{ "event_id" : 8, "genre_id" : 1, "title" : "あつた蓬莱軒 本店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000008"), "update_time" : ISODate("2013-08-18T08:59:46.759Z"), "created_time" : ISODate("2013-08-18T08:59:46.759Z"), "__v" : 0 }
{ "event_id" : 13, "genre_id" : 1, "title" : "ピッツエリア エ トラットリア ダ イーサ", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000d"), "update_time" : ISODate("2013-08-18T08:59:46.761Z"), "created_time" : ISODate("2013-08-18T08:59:46.761Z"), "__v" : 0 }
{ "event_id" : 18, "genre_id" : 1, "title" : "梅田はがくれ 本店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000012"), "update_time" : ISODate("2013-08-18T08:59:46.763Z"), "created_time" : ISODate("2013-08-18T08:59:46.763Z"), "__v" : 0 }
{ "event_id" : 2, "genre_id" : 1, "title" : "ミート矢澤", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000002"), "update_time" : ISODate("2013-08-18T08:59:46.756Z"), "created_time" : ISODate("2013-08-18T08:59:46.756Z"), "__v" : 0 }
{ "event_id" : 7, "genre_id" : 1, "title" : "イデミ・スギノ", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000007"), "update_time" : ISODate("2013-08-18T08:59:46.758Z"), "created_time" : ISODate("2013-08-18T08:59:46.758Z"), "__v" : 0 }
{ "event_id" : 12, "genre_id" : 1, "title" : "出町ふたば", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000c"), "update_time" : ISODate("2013-08-18T08:59:46.761Z"), "created_time" : ISODate("2013-08-18T08:59:46.761Z"), "__v" : 0 }
{ "event_id" : 17, "genre_id" : 1, "title" : "たいめいけん", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000011"), "update_time" : ISODate("2013-08-18T08:59:46.763Z"), "created_time" : ISODate("2013-08-18T08:59:46.763Z"), "__v" : 0 }
{ "event_id" : 1, "genre_id" : 1, "title" : "六厘舎TOKYO", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000001"), "update_time" : ISODate("2013-08-18T08:59:46.753Z"), "created_time" : ISODate("2013-08-18T08:59:46.752Z"), "__v" : 0 }
{ "event_id" : 6, "genre_id" : 1, "title" : "エッグスンシングス 原宿店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000006"), "update_time" : ISODate("2013-08-18T08:59:46.758Z"), "created_time" : ISODate("2013-08-18T08:59:46.758Z"), "__v" : 0 }
{ "event_id" : 11, "genre_id" : 1, "title" : "うさぎや", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d00000b"), "update_time" : ISODate("2013-08-18T08:59:46.760Z"), "created_time" : ISODate("2013-08-18T08:59:46.760Z"), "__v" : 0 }
{ "event_id" : 16, "genre_id" : 1, "title" : "つるとんたん 六本木店", "image" : "", "description" : "", "_id" : ObjectId("52108d02514162c71d000010"), "update_time" : ISODate("2013-08-18T08:59:46.763Z"), "created_time" : ISODate("2013-08-18T08:59:46.763Z"), "__v" : 0 }
zuqqhi2

Share
Published by
zuqqhi2
11 years ago

Recent Posts

The Benefits of PubSubClient and Points to Be Aware Of

When I send MQT…

3 months ago

Memo of How to Build Java Develop Environment

Since I don't u…

5 months ago

How to Upload Program to Arduino Using PlatformIO IDE for VSCode

When you want t…

11 months ago

How to avoid GPG error when using ROS Docker image

Some weeks ago,…

12 months ago

Trying to Develop Visited Countries Colored Map Generation App with Streamlit

Streamlit is a …

1 year ago

Run Amazon FreeRTOS on M5Stack Core2 for AWS – ESP32 IoT Development Kit で Amazon FreeRTOS (macOS)

I bought M5Stac…

2 years ago