Logging Rainforest RAVEn/EMU-2 Data to ThingSpeak (node.js on Raspberry Pi)

Last year, I bought a Rainforest EMU-2 energy monitor.  It wirelessly talks via zigbee to the smart energy meter that SCE installed at my house a couple of years ago. Unfortunately, its logging capabilities are rather primitive. All it does is give you a crude bar graph of the current and previous day’s usage. SCE actually has fairly detailed logs on their website, which you can access without buying your own energy monitor, but it’s cumbersome to log into their website and navigate down to the pages. Fortunately, a nice feature of the EMU-2 is that it has a USB port which behaves exactly like the Rainforest RAVEn, so you can read out the data into a computer and log it. I decided to hook up the EMU-2 to a Raspberry Pi, and log the data to ThingSpeak with node.js.

Step 1: Install node.js on Raspberry Pi

Unfortunately, the version of node.js that installs via apt-get on the Adafruit Occidentalis distro that I run on my Raspberry Pi is rather old (v 0.6.19), and won’t work with our code. I had to manually install from the nodejs repository:

wget http://nodejs.org/dist/v0.10.22/node-v0.10.22-linux-arm-pi.tar.gz
tar xzf node-v0.10.22-linux-arm-pi.tar.gz
cp -R node-v0.10.22-linux-arm-pi/bin /usr/local
cp -R node-v0.10.22-linux-arm-pi/lib /usr/local
cp -R node-v0.10.22-linux-arm-pi/share /usr/local
echo export NODE_PATH=/usr/local/lib/node_modules:/usr/local/lib/node_modules/npm/node_modules >> ~/.bashrc
source ~/.bashrc

I’m using node.js v0.10.22 because it works, and that’s the version that I’m using on my Debian development machine. I tried to install the latest, v0.11.11, but ran into all sorts of errors.

Step 2: Install node.js libraries

My RAVEn logging script depends on some libraries, serialportthingspeakclient, and xml2js, which we can install via npm:

npm install -g serialport thingspeakclient xml2js

Step 3: node.js code

The RAVEn XML API is fairly simple. To make things even simpler, I found that stormboy had already written a node.js library to decode it and github: stormboy/node-raven. node-raven reads data from RAVEn and uploads it to an MQTT server, so I had to modify the code to upload to ThingSpeak. Below is my script:

[code language=”java”]
/**
* Reads energy data from a smart meter via a RAVEn RFA-Z106 dongle (http://www.rainforestautomation.com/raven) and uploads to ThingSpeak.
hacked from stormboy’s node-raven https://github.com/stormboy/node-raven
by Sam C. Lin
*/

var serialport = require("serialport"),
ThingSpeakClient = require(‘thingspeakclient’),
xml2js = require(‘xml2js’);

process.on(‘uncaughtException’, function(err) {
// handle the error safely
console.log(err);
});

var TRACE = true;

// RAVEn’s serial port
var ravenSerialPath = ‘/dev/ttyUSB0’;

// thingspeak parameters
var channelId = YOUR-THINGSPEAK-CHANNELID;
var apiKey = ‘YOUR-THINGSPEAK-WRITE-API-KEY’;

var tsclient = new ThingSpeakClient();
tsclient.attachChannel(channelId, { writeKey:apiKey});

// date offset for RAVEn which presents timestamp as seconds since 2000-01-01
var dateOffset = Date.UTC(2000, 0, 1);

var dailyNet = 0;
var dailyNetSentDate = 0;

var Raven = function(serialPath) {
var self = this;

// configure the serial port that the RAVEn USB dongle is on.
this.serialPort = new serialport.SerialPort(serialPath, {
baudrate: 115200,
databits: 8,
stopbits: 1,
parity: ‘none’,
parser: serialport.parsers.readline("\r\n")
});

this.serialPort.on("open", function() {
openHandler(self);
});
};

/**
* Get the connection status between the USB device and the power meter
*/
Raven.prototype.getConnectionStatus = function() {
var queryCommand = "<Command><Name>get_connection_status</Name></Command>\r\n";
this.serialPort.write(queryCommand);
};

/**
* Get information about the device
*/
Raven.prototype.getDeviceInfo = function() {
var queryCommand = "<Command><Name>get_device_info</Name></Command>\r\n";
this.serialPort.write(queryCommand);
};

/**
* Query the amount of energy used or fed-in.
*/
Raven.prototype.getSumEnergy = function() {
var queryCommand = "<Command><Name>get_current_summation_delivered</Name></Command>\r\n";
this.serialPort.write(queryCommand);
};

/**
* Get the power currently being used (or fed-in)
*/
Raven.prototype.getSumPower = function() {
var queryCommand = "<Command><Name>get_instantaneous_demand</Name></Command>\r\n";
this.serialPort.write(queryCommand);
};

Raven.prototype.getMessage = function() {
var queryCommand = "<Command><Name>get_message</Name></Command>\r\n";
this.serialPort.write(queryCommand);
};

Raven.prototype.getTime = function() {
var queryCommand = "<Command><Name>get_time</Name></Command>\r\n";
this.serialPort.write(queryCommand);
};

Raven.prototype.getCurrentPrice = function() {
var queryCommand = "<Command><Name>get_current_price</Name></Command>\r\n";
this.serialPort.write(queryCommand);
};

Raven.prototype.close = function() {
this.serialPort.close();
};

// handle serial port open
function openHandler (self) {
var parser = new xml2js.Parser();
var buffer = ""; // read buffer.

if (TRACE) {
console.log(‘serial device open’);
}

// add serial port data handler
self.serialPort.on(‘data’, function(data) {
buffer += data.toString() + "\r\n"; // append to the read buffer
if ( data.toString().indexOf(‘</’) == 0 ) { // check if last part of XML element.

// try to parse buffer
parser.parseString(buffer, function (err, result) {
if (err) {
console.log("err: " + err);
console.log(‘data received: ‘ + buffer);
}
else if (result.InstantaneousDemand) {
var timestamp = parseInt( result.InstantaneousDemand.TimeStamp );
timestamp = new Date(dateOffset+timestamp*1000);
var demand = parseInt( result.InstantaneousDemand.Demand, 16 );
demand = demand < 0x80000000 ? demand : – ~demand – 1;
if (TRACE) {
console.log("demand: " + timestamp.toLocaleString() + " : " + demand);
}
var tsData = new Object();
tsData = { field1: demand };
tsclient.updateChannel(channelId,tsData);
}
else if (result.CurrentSummationDelivered) {
var timestamp = parseInt( result.CurrentSummationDelivered.TimeStamp );
timestamp = new Date(dateOffset+timestamp*1000);
var used = parseInt( result.CurrentSummationDelivered.SummationDelivered, 16 );
var fedin = parseInt( result.CurrentSummationDelivered.SummationReceived, 16 );
var curDate = timestamp.getDate();
var net = used – fedin;

if (dailyNet == 0) {
dailyNet = net;
dailyNetSentDate = curDate;
}

if (TRACE) {
console.log("sum: " + timestamp.toLocaleString() + " : " + used + " – " + fedin);
}

var tsData = new Object();
tsData = { field2: net,field3: used,field4: fedin};

// only send daily net once a day
if (curDate !== dailyNetSentDate) {
tsData.field5 = net – dailyNet;
dailyNet = net;
dailyNetSentDate = curDate;
}

tsclient.updateChannel(channelId,tsData);

}
else if (result.ConnectionStatus) {
if (TRACE) {
console.log("connection status: " + result.ConnectionStatus.Status);
}
}
else {
if (TRACE) {
console.dir(result); // display data read in
}
}
});
buffer = ""; // reset the read buffer
}
});
}

var raven = Raven(ravenSerialPath);
[/code]

Before you can run the script, you must ravenSerialPath, channelId, and apiKey to match your own configuration. Node that the code contains several getters which I don’t use, because by default, my EMU-2 sends out the data via the USB serial port at regular intervals.

Step 4: ThingSpeak Channel Configuration

Next, you must create a new ThingSpeak channel with 5 fields:
tsraven

Step 5: Run our node.js script

node raven-log.js &

Below are the live data from my ThingSpeak channel: Rainforest RAVEn Logging Demo






I will be updating the code from time to time. You can always get the latest version from github: lincomatic/raven-thingspeak

Logging Nest Thermostat Data: Update 1

I have updated my Nest thermostat data logger. The changes are as follows:

  1. added fan status, heater status, and AC status.
  2. polling interval increased from 15 min to 1 min.
  3. data are pushed to ThingSpeak only if at least one field changes.

The new node.js code:

[code language=”java”]
"option strict";
var util = require(‘util’),
ThingSpeakClient = require(‘thingspeakclient’);
nest = require(‘unofficial-nest-api’); // get from npm

process.on(‘uncaughtException’, function(err) {
// handle the error safely
console.log(err);
});

// nest parameters
var username = ‘YOUR-NEST-LOGIN’;
var password = ‘YOUR NEST PASSWORD’;

// thingspeak parameters
var channelId = YOUR-THINGSPEAK-CHANNEL-ID;
var apiKey = ‘YOUR THINGSPEAK-WRITE-API-KEY’;

// update interval in ms
var updateInterval = 1000*60;

var tsclient = new ThingSpeakClient();
tsclient.attachChannel(channelId, { writeKey:apiKey});

var setTemp = 0;
var curTemp = 0;
var curHum = 0;
var curFanState = -1;
var curHeaterState = -1;
var curACState = -1;

function trimQuotes(s) {
if (!s || s.length === 0) {
return ”;
}
var c = s.charAt(0);
var start = (c === ‘\” || c === ‘"’) ? 1 : 0;
var end = s.length;
c = s.charAt(end – 1);
end -= (c === ‘\” || c === ‘"’) ? 1 : 0;
return s.substring(start, end);
}

function merge(o1, o2) {
o1 = o1 || {};
if (!o2) {
return o1;
}
for (var p in o2) {
o1[p] = o2[p];
}
return o1;
}

function fetchData(data) {
nest.login(username, password, function (err, data) {
if (err) {
console.log(err.message);
//process.exit(1);
return;
}

nest.fetchStatus(function (data) {
for (var deviceId in data.device) {
if (data.device.hasOwnProperty(deviceId)) {
var shared = data.shared[deviceId];
var date = new Date();
var time = date.getFullYear()+’/’+date.getMonth()+’/’+date.getDate()+’-‘+date.getHours()+’:’+date.getMinutes();
var cTemp = nest.ctof(shared.current_temperature);
var sTemp = nest.ctof(shared.target_temperature);
var cHum = data.device[deviceId].current_humidity;
var cFanState = (shared.hvac_fan_state == true) ? 1 : 0;
var cHeaterState = (shared.hvac_heater_state == true) ? 1 : 0;
var cACState = (shared.hvac_ac_state == true) ? 1 : 0;

console.log(util.format("%s %s [%s], cur temp: %d F cur humidity: %d %% set temp: %d fan: %s heat: %s AC: %s",
time,
shared.name, deviceId,
cTemp,
cHum,
sTemp,
cFanState ? ‘on’ : ‘off’,
cHeaterState ? ‘on’ : ‘off’,
cACState ? ‘on’ : ‘off’
));
if ((cTemp !== curTemp) || (cHum !== curHum) ||
(sTemp !== setTemp) || (cFanState !== curFanState) ||
(cHeaterState !== curHeaterState) ||
(cACState !== curACState)) {
var tsData = new Object();
tsData.field1 = cTemp;
tsData.field2 = cHum;
tsData.field3 = sTemp;
tsData.field4 = cFanState;
tsData.field5 = cHeaterState;
tsData.field6 = cACState;
console.log("sending to thingspeak");
tsclient.updateChannel(channelId,tsData);
}
curTemp = cTemp;
curHum = cHum;
setTemp = sTemp;
curFanState = cFanState;
curHeaterState = cHeaterState;
curACState = cACState;
}
}
});
});
}

fetchData();
setInterval(fetchData,updateInterval);
[/code]

The corresponding ThingSpeak Channel also needs to have 3 new fields added to it:

  1. Field 4: Fan State
  2. Field 5: Heater State
  3. Field 6: AC State

I am currently running the demo on a Raspberry Pi. As before, you can view my live channel on ThingSpeak: Nest Logging Demo

I will be updating the code from time to time.  You can always get the latest version from github: lincomatic/nest-thingspeak

Related Articles: Nest

Logging Nest Thermostat Data

I recently had a Nest Thermostat installed at my house for free by my gas company as part of a pilot program that they’re running. It’s a pretty cool device, though I wouldn’t spend $250 to buy one. I like the fact that it’s a lot easier to use than my previous smart thermostat, especially being able to program it from a computer or my smartphone.  One very interesting feature to me is that it tracks the relative humidity of my house. Unfortunately, the Nest app doesn’t let you keep a running log of your data, so I decided to hack together a solution today.

I used node.js to implement a data logger which queries some parameters from my Nest thermostat, and upload it to ThingSpeak at regular intervals. Specifically, my node.js script uploads my Nest’s current temperature, humidity, and the current set temperature to thingspeak every 15 minutes. Below is the node.js script:

[code language=”java”]
"option strict";
var util = require(‘util’),
ThingSpeakClient = require(‘thingspeakclient’);
nest = require(‘unofficial-nest-api’); // get from npm

// nest parameters
var username = ‘YOUR-NEST-USERNAME’;
var password = ‘YOUR-NEST-PASSWORD’;

// thingspeak parameters
var channelId = YOUR-THINGSPEAK-CHANNELID;
var apiKey = ‘YOUR-THINGSPEAK-WRITE-API-KEY’;

// update interval in ms
var updateInterval = 1000*60*15;

var tsclient = new ThingSpeakClient();
tsclient.attachChannel(channelId, { writeKey:apiKey});

function trimQuotes(s) {
if (!s || s.length === 0) {
return ”;
}
var c = s.charAt(0);
var start = (c === ‘\” || c === ‘"’) ? 1 : 0;
var end = s.length;
c = s.charAt(end – 1);
end -= (c === ‘\” || c === ‘"’) ? 1 : 0;
return s.substring(start, end);
}

function merge(o1, o2) {
o1 = o1 || {};
if (!o2) {
return o1;
}
for (var p in o2) {
o1[p] = o2[p];
}
return o1;
}

function fetchData(data) {
nest.login(username, password, function (err, data) {
if (err) {
console.log(err.message);
process.exit(1);
return;
}

nest.fetchStatus(function (data) {
for (var deviceId in data.device) {
if (data.device.hasOwnProperty(deviceId)) {
var shared = data.shared[deviceId];
var date = new Date();
var time = date.getFullYear()+’/’+date.getMonth()+’/’+date.getDate()+’-‘+date.getHours()+’:’+date.getMinutes();

console.log(util.format("%s %s [%s], Current temp: %d F Current Humidity: %d %% Target temp: %d",
time,
shared.name, deviceId,

nest.ctof(shared.current_temperature),
data.device[deviceId].current_humidity,

nest.ctof(shared.target_temperature)));

var fields = { field1: nest.ctof(shared.current_temperature),field2: data.device[deviceId].current_humidity, field3: nest.ctof(shared.target_temperature)};
tsclient.updateChannel(channelId,fields);
}
}
});
});
}

fetchData();
setInterval(fetchData,updateInterval);
[/code]

In order to run my node.js script, you need to first use npm to install unofficial-nest-api and thingspeakclient. My script is actually just a hacked up version of the example that comes with unofficial-nest-api.

Note that you must first update the initialization of username and password with your Nest.com login credentials.

Also, you need to enter your channel id and write api key from thingspeak.com. Below are my thingspeak channel settings:

channel

You can view my channel live on thingspeak here: Nest Logging Demo

channelview