Changes for NSE #7
Description
`/*
This file is a node.js module.
This is a sample implementation of UDF-compatible datafeed wrapper for Quandl (historical data) and yahoo.finance (quotes).
Some algorithms may be incorrect because it's rather an UDF implementation sample
then a proper datafeed implementation.
*/
/* global require /
/ global console /
/ global exports /
/ global process */
"use strict";
var version = '2.0.2';
var https = require("https");
var http = require("http");
var quandlCache = {};
var quandlCacheCleanupTime = 3 * 60 * 60 * 1000; // 3 hours
var quandlKeysValidateTime = 15 * 60 * 1000; // 15 minutes
var yahooFailedStateCacheTime = 3 * 60 * 60 * 1000; // 3 hours;
var quandlMinimumDate = '1970-01-01';
// this cache is intended to reduce number of requests to Quandl
setInterval(function () {
quandlCache = {};
console.warn(dateForLogs() + 'Quandl cache invalidated');
}, quandlCacheCleanupTime);
function dateForLogs() {
return (new Date()).toISOString() + ': ';
}
var defaultResponseHeader = {
"Content-Type": "text/plain",
'Access-Control-Allow-Origin': '*'
};
function sendJsonResponse(response, jsonData) {
response.writeHead(200, defaultResponseHeader);
response.write(JSON.stringify(jsonData));
response.end();
}
function dateToYMD(date) {
var obj = new Date(date);
var year = obj.getFullYear();
var month = obj.getMonth() + 1;
var day = obj.getDate();
return year + "-" + month + "-" + day;
}
var quandlKeys = process.env.QUANDL_API_KEY.split(','); // you should create a free account on quandl.com to get this key, you can set some keys concatenated with a comma
var invalidQuandlKeys = [];
function getValidQuandlKey() {
for (var i = 0; i < quandlKeys.length; i++) {
var key = quandlKeys[i];
if (invalidQuandlKeys.indexOf(key) === -1) {
return key;
}
}
return null;
}
function markQuandlKeyAsInvalid(key) {
if (invalidQuandlKeys.indexOf(key) !== -1) {
return;
}
invalidQuandlKeys.push(key);
console.warn(dateForLogs() + 'Quandl key invalidated ' + key);
setTimeout(function() {
console.log(dateForLogs() + "Quandl key restored: " + invalidQuandlKeys.shift());
}, quandlKeysValidateTime);
}
function sendError(error, response) {
response.writeHead(200, defaultResponseHeader);
response.write("{"s":"error","errmsg":"" + error + ""}");
response.end();
}
function httpGet(datafeedHost, path, callback) {
var options = {
host: datafeedHost,
path: path
};
function onDataCallback(response) {
var result = '';
response.on('data', function (chunk) {
result += chunk;
});
response.on('end', function () {
if (response.statusCode !== 200) {
callback({ status: 'ERR_STATUS_CODE', errmsg: response.statusMessage || '' });
return;
}
callback({ status: 'ok', data: result });
});
}
var req = https.request(options, onDataCallback);
req.on('socket', function (socket) {
socket.setTimeout(5000);
socket.on('timeout', function () {
console.log(dateForLogs() + 'timeout');
req.abort();
});
});
req.on('error', function (e) {
callback({ status: 'ERR_SOCKET', errmsg: e.message || '' });
});
req.end();
}
function convertQuandlHistoryToUDFFormat(data) {
function parseDate(input) {
var parts = input.split('-');
return Date.UTC(parts[0], parts[1] - 1, parts[2]);
}
function columnIndices(columns) {
var indices = {};
for (var i = 0; i < columns.length; i++) {
indices[columns[i].name] = i;
}
return indices;
}
console.log(data);
var result = {
t: [],
c: [],
o: [],
h: [],
l: [],
v: [],
s: "ok"
};
try {
var json = JSON.parse(data);
var datatable = json.datatable;
var idx = columnIndices(datatable.columns);
datatable.data.forEach(function (row) {
result.t.push(parseDate(row[idx.date]) / 1000);
result.o.push(row[idx.open]);
result.h.push(row[idx.high]);
result.l.push(row[idx.low]);
result.c.push(row[idx.close]);
result.v.push(row[idx.volume]);
});
} catch (error) {
return null;
}
return result;
}
function MyConvertQuandlHistoryToUDFFormat(data) {
function parseDate(input) {
var parts = input.split('-');
return Date.UTC(parts[0], parts[1] - 1, parts[2]);
}
function columnIndices(columns) {
var indices = {};
for (var i = 0; i < columns.length; i++) {
indices[columns[i].name] = i;
}
return indices;
}
//console.log(data);
var result = {
t: [],
c: [],
o: [],
h: [],
l: [],
v: [],
s: "ok"
};
try {
var json = JSON.parse(data);
var datatable = json.dataset;
//var idx = columnIndices(datatable.column_names);
//console.log(idx);
datatable.data.forEach(function (row) {
result.t.push(parseDate(row[0]) / 1000);
result.o.push(row[1]);
result.h.push(row[2]);
result.l.push(row[3]);
result.c.push(row[4]);
result.v.push(row[5]);
});
} catch (error) {
return null;
}
return result;
}
function convertYahooQuotesToUDFFormat(tickersMap, data) {
if (!data.query || !data.query.results) {
var errmsg = "ERROR: empty quotes response: " + JSON.stringify(data);
console.log(dateForLogs() + errmsg);
return {
s: "error",
errmsg: errmsg
};
}
var result = {
s: "ok",
d: []
};
[].concat(data.query.results.quote).forEach(function (quote) {
var ticker = tickersMap[quote.symbol];
// this field is an error token
if (quote["ErrorIndicationreturnedforsymbolchangedinvalid"] || !quote.StockExchange) {
result.d.push({
s: "error",
n: ticker,
v: {}
});
return;
}
result.d.push({
s: "ok",
n: ticker,
v: {
ch: +(quote.ChangeRealtime || quote.Change),
chp: +((quote.PercentChange || quote.ChangeinPercent) && (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1")),
short_name: quote.Symbol,
exchange: quote.StockExchange,
original_name: quote.StockExchange + ":" + quote.Symbol,
description: quote.Name,
lp: +quote.LastTradePriceOnly,
ask: +quote.AskRealtime,
bid: +quote.BidRealtime,
open_price: +quote.Open,
high_price: +quote.DaysHigh,
low_price: +quote.DaysLow,
prev_close_price: +quote.PreviousClose,
volume: +quote.Volume,
}
});
});
return result;
}
function proxyRequest(controller, options, response) {
controller.request(options, function (res) {
var result = '';
res.on('data', function (chunk) {
result += chunk;
});
res.on('end', function () {
if (res.statusCode !== 200) {
response.writeHead(200, defaultResponseHeader);
response.write(JSON.stringify({
s: 'error',
errmsg: 'Failed to get news'
}));
response.end();
return;
}
response.writeHead(200, defaultResponseHeader);
response.write(result);
response.end();
});
}).end();
}
function RequestProcessor(symbolsDatabase) {
this._symbolsDatabase = symbolsDatabase;
this._failedYahooTime = {};
}
function filterDataPeriod(data, fromSeconds, toSeconds) {
if (!data || !data.t) {
return data;
}
if (data.t[data.t.length - 1] < fromSeconds) {
return {
s: 'no_data',
nextTime: data.t[data.t.length - 1]
};
}
var fromIndex = null;
var toIndex = null;
var times = data.t;
for (var i = 0; i < times.length; i++) {
var time = times[i];
if (fromIndex === null && time >= fromSeconds) {
fromIndex = i;
}
if (toIndex === null && time >= toSeconds) {
toIndex = time > toSeconds ? i - 1 : i;
}
if (fromIndex !== null && toIndex !== null) {
break;
}
}
fromIndex = fromIndex || 0;
toIndex = toIndex ? toIndex + 1 : times.length;
var s = data.s;
if (toSeconds < times[0]) {
s = 'no_data';
}
toIndex = Math.min(fromIndex + 1000, toIndex); // do not send more than 1000 bars for server capacity reasons
return {
t: data.t.slice(fromIndex, toIndex),
o: data.o.slice(fromIndex, toIndex),
h: data.h.slice(fromIndex, toIndex),
l: data.l.slice(fromIndex, toIndex),
c: data.c.slice(fromIndex, toIndex),
v: data.v.slice(fromIndex, toIndex),
s: s
};
}
RequestProcessor.prototype._sendConfig = function (response) {
var config = {
supports_search: true,
supports_group_request: false,
supports_marks: true,
supports_timescale_marks: true,
supports_time: true,
exchanges: [
{
value: "",
name: "All Exchanges",
desc: ""
},
{
value: "NasdaqNM",
name: "NasdaqNM",
desc: "NasdaqNM"
},
{
value: "NYSE",
name: "NYSE",
desc: "NYSE"
},
{
value: "NCM",
name: "NCM",
desc: "NCM"
},
{
value: "NGM",
name: "NGM",
desc: "NGM"
},
{
value: "NSE",
name: "NSE",
desc: "NSE"
}],
symbols_types: [
{
name: "All types",
value: ""
},
{
name: "Stock",
value: "stock"
},
{
name: "Index",
value: "index"
}
],
supported_resolutions: ["D", "2D", "3D", "W", "3W", "M", '6M']
};
response.writeHead(200, defaultResponseHeader);
response.write(JSON.stringify(config));
response.end();
};
RequestProcessor.prototype._sendMarks = function (response) {
var now = new Date();
now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000;
var day = 60 * 60 * 24;
var marks = {
id: [0, 1, 2, 3, 4, 5],
time: [now, now - day * 4, now - day * 7, now - day * 7, now - day * 15, now - day * 30],
color: ["red", "blue", "green", "red", "blue", "green"],
text: ["Today", "4 days back", "7 days back + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", "7 days back once again", "15 days back", "30 days back"],
label: ["A", "B", "CORE", "D", "EURO", "F"],
labelFontColor: ["white", "white", "red", "#FFFFFF", "white", "#000"],
minSize: [14, 28, 7, 40, 7, 14]
};
response.writeHead(200, defaultResponseHeader);
response.write(JSON.stringify(marks));
response.end();
};
RequestProcessor.prototype._sendTime = function (response) {
var now = new Date();
response.writeHead(200, defaultResponseHeader);
response.write(Math.floor(now / 1000) + '');
response.end();
};
RequestProcessor.prototype._sendTimescaleMarks = function (response) {
var now = new Date();
now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000;
var day = 60 * 60 * 24;
var marks = [
{
id: "tsm1",
time: now,
color: "red",
label: "A",
tooltip: ""
},
{
id: "tsm2",
time: now - day * 4,
color: "blue",
label: "D",
tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()]
},
{
id: "tsm3",
time: now - day * 7,
color: "green",
label: "D",
tooltip: ["Dividends: $3.46", "Date: " + new Date((now - day * 7) * 1000).toDateString()]
},
{
id: "tsm4",
time: now - day * 15,
color: "#999999",
label: "E",
tooltip: ["Earnings: $3.44", "Estimate: $3.60"]
},
{
id: "tsm7",
time: now - day * 30,
color: "red",
label: "E",
tooltip: ["Earnings: $5.40", "Estimate: $5.00"]
},
];
response.writeHead(200, defaultResponseHeader);
response.write(JSON.stringify(marks));
response.end();
};
RequestProcessor.prototype._sendSymbolSearchResults = function (query, type, exchange, maxRecords, response) {
if (!maxRecords) {
throw "wrong_query";
}
var result = this._symbolsDatabase.search(query, type, exchange, maxRecords);
response.writeHead(200, defaultResponseHeader);
response.write(JSON.stringify(result));
response.end();
};
RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) {
var symbolInfo = this._symbolsDatabase.symbolInfo(symbolName);
if (!symbolInfo) {
throw "unknown_symbol " + symbolName;
}
return {
"name": symbolInfo.name,
"exchange-traded": symbolInfo.exchange,
"exchange-listed": symbolInfo.exchange,
"timezone": "Asia/Kolkata",
"minmov": 1,
"minmov2": 0,
"pointvalue": 1,
"session": "0800-1630",
"has_intraday": false,
"has_no_volume": symbolInfo.type !== "stock",
"description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name,
"type": symbolInfo.type,
"supported_resolutions": ["D", "2D", "3D", "W", "3W", "M", "6M"],
"pricescale": 100,
"ticker": symbolInfo.name.toUpperCase()
};
};
RequestProcessor.prototype._sendSymbolInfo = function (symbolName, response) {
var info = this._prepareSymbolInfo(symbolName);
response.writeHead(200, defaultResponseHeader);
response.write(JSON.stringify(info));
response.end();
};
RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimestamp, endDateTimestamp, resolution, response) {
function sendResult(content) {
var header = Object.assign({}, defaultResponseHeader);
header["Content-Length"] = content.length;
response.writeHead(200, header);
response.write(content, null, function () {
response.end();
});
}
function secondsToISO(sec) {
if (sec === null || sec === undefined) {
return 'n/a';
}
return (new Date(sec * 1000).toISOString());
}
function logForData(data, key, isCached) {
var fromCacheTime = data && data.t ? data.t[0] : null;
var toCacheTime = data && data.t ? data.t[data.t.length - 1] : null;
console.log(dateForLogs() + "Return QUANDL result" + (isCached ? " from cache" : "") + ": " + key + ", from " + secondsToISO(fromCacheTime) + " to " + secondsToISO(toCacheTime));
}
console.log(dateForLogs() + "Got history request for " + symbol + ", " + resolution + " from " + secondsToISO(startDateTimestamp)+ " to " + secondsToISO(endDateTimestamp));
// always request all data to reduce number of requests to quandl
var from = quandlMinimumDate;
var to = dateToYMD(Date.now());
var key = symbol + "|" + from + "|" + to;
if (quandlCache[key]) {
var dataFromCache = filterDataPeriod(quandlCache[key], startDateTimestamp, endDateTimestamp);
logForData(dataFromCache, key, true);
sendResult(JSON.stringify(dataFromCache));
return;
}
var quandlKey = getValidQuandlKey();
if (quandlKey === null) {
console.log(dateForLogs() + "No valid quandl key available");
sendError('No valid API Keys available', response);
return;
}
var address = "/api/v3/datasets/NSE/"+symbol+
".json?api_key="+quandlKey+
"&start_date="+from+
"&end_date="+to+"";
console.log(dateForLogs() + "Sending request to quandl " + key + ". url="+address+"");
httpGet("www.quandl.com", address, function (result) {
if (response.finished) {
// we can be here if error happened on socket disconnect
return;
}
if (result.status !== 'ok') {
if (result.status === 'ERR_SOCKET') {
console.log('Socket problem with request: ' + result.errmsg);
sendError("Socket problem with request " + result.errmsg, response);
return;
}
console.error(dateForLogs() + "Error response from quandl for key " + key + ". Message: " + result.errmsg);
markQuandlKeyAsInvalid(quandlKey);
sendError("Error quandl response " + result.errmsg, response);
return;
}
console.log(dateForLogs() + "Got response from quandl " + key + ". Try to parse.");
var data = MyConvertQuandlHistoryToUDFFormat(result.data);
//console.log(data);
//response.write(JSON.stringify(convertQuandlHistoryToUDFFormat(result.data)));
if (data === null) {
var dataStr = typeof result === "string" ? result.slice(0, 100) : result;
console.error(dateForLogs() + " failed to parse: " + dataStr);
sendError("Invalid quandl response", response);
return;
}
if (data.t.length !== 0) {
console.log(dateForLogs() + "Successfully parsed and put to cache " + data.t.length + " bars.");
quandlCache[key] = data;
} else {
console.log(dateForLogs() + "Parsing returned empty result.");
}
var filteredData = filterDataPeriod(data, startDateTimestamp, endDateTimestamp);
console.log(filteredData);
logForData(filteredData, key, false);
sendResult(JSON.stringify(filteredData));
});
};
RequestProcessor.prototype._quotesQuandlWorkaround = function (tickersMap) {
var from = quandlMinimumDate;
var to = dateToYMD(Date.now());
var result = {
s: "ok",
d: [],
source: 'Quandl',
};
Object.keys(tickersMap).forEach(function(symbol) {
var key = symbol + "|" + from + "|" + to;
var ticker = tickersMap[symbol];
var data = quandlCache[key];
var length = data === undefined ? 0 : data.c.length;
if (length > 0) {
var lastBar = {
o: data.o[length - 1],
h: data.o[length - 1],
l: data.o[length - 1],
c: data.o[length - 1],
v: data.o[length - 1],
};
result.d.push({
s: "ok",
n: ticker,
v: {
ch: 0,
chp: 0,
short_name: symbol,
exchange: '',
original_name: ticker,
description: ticker,
lp: lastBar.c,
ask: lastBar.c,
bid: lastBar.c,
open_price: lastBar.o,
high_price: lastBar.h,
low_price: lastBar.l,
prev_close_price: length > 1 ? data.c[length - 2] : lastBar.o,
volume: lastBar.v,
}
});
}
});
return result;
};
RequestProcessor.prototype._sendQuotes = function (tickersString, response) {
var tickersMap = {}; // maps YQL symbol to ticker
var tickers = tickersString.split(",");
[].concat(tickers).forEach(function (ticker) {
var yqlSymbol = ticker.replace(/.*:(.*)/, "$1");
tickersMap[yqlSymbol] = ticker;
});
if (this._failedYahooTime[tickersString] && Date.now() - this._failedYahooTime[tickersString] < yahooFailedStateCacheTime) {
sendJsonResponse(response, this._quotesQuandlWorkaround(tickersMap));
console.log("Quotes request : " + tickersString + ' processed from quandl cache');
return;
}
var that = this;
var yql = "env 'store://datatables.org/alltableswithkeys'; select * from yahoo.finance.quotes where symbol in ('" + Object.keys(tickersMap).join("','") + "')";
console.log("Quotes query: " + yql);
var options = {
host: "query.yahooapis.com",
path: "/v1/public/yql?q=" + encodeURIComponent(yql) +
"&format=json" +
"&env=store://datatables.org/alltableswithkeys"
};
// for debug purposes
// console.log(options.host + options.path);
http.request(options, function (res) {
var result = '';
res.on('data', function (chunk) {
result += chunk;
});
res.on('end', function () {
var jsonResponse = { s: 'error' };
if (res.statusCode === 200) {
jsonResponse = convertYahooQuotesToUDFFormat(tickersMap, JSON.parse(result));
} else {
console.error('Yahoo Fails with code ' + res.statusCode);
}
if (jsonResponse.s === 'error') {
that._failedYahooTime[tickersString] = Date.now();
jsonResponse = that._quotesQuandlWorkaround(tickersMap);
console.log("Quotes request : " + tickersString + ' processed from quandl');
}
sendJsonResponse(response, jsonResponse);
});
}).end();
};
RequestProcessor.prototype._sendNews = function (symbol, response) {
var options = {
host: "feeds.finance.yahoo.com",
path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US"
};
proxyRequest(https, options, response);
};
RequestProcessor.prototype._sendFuturesmag = function (response) {
var options = {
host: "www.futuresmag.com",
path: "/rss/all"
};
proxyRequest(http, options, response);
};
RequestProcessor.prototype.processRequest = function (action, query, response) {
try {
if (action === "/config") {
this._sendConfig(response);
}
else if (action === "/symbols" && !!query["symbol"]) {
this._sendSymbolInfo(query["symbol"], response);
}
else if (action === "/search") {
this._sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response);
}
else if (action === "/history") {
this._sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response);
}
else if (action === "/quotes") {
this._sendQuotes(query["symbols"], response);
}
else if (action === "/marks") {
this._sendMarks(response);
}
else if (action === "/time") {
this._sendTime(response);
}
else if (action === "/timescale_marks") {
this._sendTimescaleMarks(response);
}
else if (action === "/news") {
this._sendNews(query["symbol"], response);
}
else if (action === "/futuresmag") {
this._sendFuturesmag(response);
} else {
response.writeHead(200, defaultResponseHeader);
response.write('Datafeed version is ' + version +
'\nValid keys count is ' + String(quandlKeys.length - invalidQuandlKeys.length) +
'\nCurrent key is ' + (getValidQuandlKey() || '').slice(0, 3) +
(invalidQuandlKeys.length !== 0 ? '\nInvalid keys are ' + invalidQuandlKeys.reduce(function(prev, cur) { return prev + cur.slice(0, 3) + ','; }, '') : ''));
response.end();
}
}
catch (error) {
sendError(error, response);
console.error('Exception: ' + error);
}
};
exports.RequestProcessor = RequestProcessor;
`
I think i make all the charges work for NSE data from quandl, but i am getting nothing on TV graph