From e15e1f058217d8f0979ec4c93694943a88350a01 Mon Sep 17 00:00:00 2001 From: Eugene Date: Mon, 2 Mar 2015 11:26:01 +0300 Subject: [PATCH 01/59] Added supported resolutions to symbol_info --- yahoo.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/yahoo.js b/yahoo.js index 5784af7..89c260d 100644 --- a/yahoo.js +++ b/yahoo.js @@ -247,7 +247,8 @@ RequestProcessor = function(action, query, response) { "has_no_volume": symbolInfo.type != "stock", "ticker": _lastYahooResponse["ticker"].toUpperCase(), "description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name, - "type": symbolInfo.type + "type": symbolInfo.type, + "supported_resolutions" : ["D","2D","3D","W","3W","M","6M"] }; response.writeHead(200, defaultResponseHeader); From 0a7ac6d0adba242ebf866ebb416247574798b998 Mon Sep 17 00:00:00 2001 From: Eugene Date: Mon, 6 Apr 2015 16:24:42 +0300 Subject: [PATCH 02/59] Fixed exception where there is not change in the quotes --- yahoo.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yahoo.js b/yahoo.js index 89c260d..67f1fb7 100644 --- a/yahoo.js +++ b/yahoo.js @@ -81,7 +81,7 @@ function convertYahooQuotesToUDFFormat(tickersMap, data) { var ticker = tickersMap[quote.symbol]; // this field is an error token - if (quote["ErrorIndicationreturnedforsymbolchangedinvalid"]) { + if (quote["ErrorIndicationreturnedforsymbolchangedinvalid"] || !quote.StockExchange) { result.d.push({ s: "error", n: ticker, v: {} }); return; } @@ -91,7 +91,7 @@ function convertYahooQuotesToUDFFormat(tickersMap, data) { n: ticker, v: { ch: quote.ChangeRealtime || quote.Change, - chp: (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1"), + chp: (quote.PercentChange || quote.ChangeinPercent) && (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1"), short_name: quote.Symbol, exchange: quote.StockExchange, From acb54195f1f861379564b3d68ebd0ad5c487080c Mon Sep 17 00:00:00 2001 From: Eugene Date: Thu, 23 Apr 2015 17:32:39 +0300 Subject: [PATCH 03/59] FIXED: duplicating timezone --- yahoo.js | 1 - 1 file changed, 1 deletion(-) diff --git a/yahoo.js b/yahoo.js index 67f1fb7..37b3c6d 100644 --- a/yahoo.js +++ b/yahoo.js @@ -241,7 +241,6 @@ RequestProcessor = function(action, query, response) { "minmov2": 0, "pricescale": pricescale, "pointvalue": 1, - "timezone": "UTC", "session": "0930-1630", "has_intraday": false, "has_no_volume": symbolInfo.type != "stock", From 7a1ff38fe9f1a7982f5929d75fb43ffef088d520 Mon Sep 17 00:00:00 2001 From: Eugene Date: Wed, 22 Jul 2015 12:40:27 +0300 Subject: [PATCH 04/59] Fixed hanging of requests --- yahoo.js | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/yahoo.js b/yahoo.js index 37b3c6d..2f73b48 100644 --- a/yahoo.js +++ b/yahoo.js @@ -33,7 +33,22 @@ function httpGet(path, callback) }); } - http.request(options, onDataCallback).end(); + var req = http.request(options, onDataCallback); + + req.on('socket', function (socket) { + socket.setTimeout(5000); + socket.on('timeout', function() { + console.log('timeout'); + req.abort(); + }); + }); + + req.on('error', function(e) { + console.log('Problem with request: ' + e.message); + callback(''); + }); + + req.end(); } From 07e5be0c4c7561c2cb4682448c90a91667320857 Mon Sep 17 00:00:00 2001 From: Eugene Date: Thu, 13 Aug 2015 09:33:01 +0300 Subject: [PATCH 05/59] Timescale marks Conflicts: yahoo.js --- yahoo.js | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/yahoo.js b/yahoo.js index 2f73b48..ddea6da 100644 --- a/yahoo.js +++ b/yahoo.js @@ -145,6 +145,7 @@ RequestProcessor = function(action, query, response) { supports_search: true, supports_group_request: false, supports_marks: true, + supports_timescale_marks: true, exchanges: [ {value: "", name: "All Exchanges", desc: ""}, {value: "XETRA", name: "XETRA", desc: "XETRA"}, @@ -186,6 +187,24 @@ RequestProcessor = function(action, query, response) { response.write(JSON.stringify(marks)); response.end(); } + + this.sendTimescaleMarks = function(response) { + var now = new Date().valueOf() / 1000; + var day = 60 * 60 * 24; + + var marks = [ + {id: "tsm1", time: now - day * 1, color: "red", label: "A", tooltip: ""}, + {id: "tsm2", time: now - day * 5, color: "blue", label: "D", tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()]}, + {id: "tsm3", time: now - day * 7, color: "green", label: "D", tooltip: ["Dividends: $3.46", "Date: " + new Date((now - day * 7) * 1000).toDateString()]}, + {id: "tsm4", time: now - day * 10, color: "#999999", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"]}, + {id: "tsm5", time: now - day * 14, color: "purple", label: "S", tooltip: ["Split: 2/1", "Date: " + new Date((now - day * 14) * 1000).toDateString()]}, + {id: "tsm6", time: now - day * 18, color: "black", label: "E", tooltip: ["Earnings: $5.40", "Estimate: $5.00"]}, + ]; + + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(marks)); + response.end(); + }; this.sendSymbolSearchResults = function(query, type, exchange, maxRecords, response) { @@ -366,8 +385,8 @@ RequestProcessor = function(action, query, response) { else if (action == "/marks") { this.sendMarks(response); } - else { - throw "wrong_request_format"; + else if (action == "/timescale_marks") { + this.sendTimescaleMarks(response); } } catch (error) { From 8401ff72e9227e3604daec9e5923a67e14a868c8 Mon Sep 17 00:00:00 2001 From: Eugene Date: Thu, 13 Aug 2015 18:23:26 +0300 Subject: [PATCH 06/59] Correct dates of marks (unix date without time) --- yahoo.js | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/yahoo.js b/yahoo.js index ddea6da..5e72356 100644 --- a/yahoo.js +++ b/yahoo.js @@ -170,7 +170,8 @@ RequestProcessor = function(action, query, response) { this.sendMarks = function(response) { - var now = new Date().valueOf() / 1000; + var now = new Date(); + now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; var day = 60 * 60 * 24; var marks = { @@ -189,16 +190,16 @@ RequestProcessor = function(action, query, response) { } this.sendTimescaleMarks = function(response) { - var now = new Date().valueOf() / 1000; + var now = new Date(); + now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; var day = 60 * 60 * 24; var marks = [ - {id: "tsm1", time: now - day * 1, color: "red", label: "A", tooltip: ""}, - {id: "tsm2", time: now - day * 5, color: "blue", label: "D", tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()]}, + {id: "tsm1", time: now - day * 0, color: "red", label: "A", tooltip: ""}, + {id: "tsm2", time: now - day * 4, color: "blue", label: "D", tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()]}, {id: "tsm3", time: now - day * 7, color: "green", label: "D", tooltip: ["Dividends: $3.46", "Date: " + new Date((now - day * 7) * 1000).toDateString()]}, - {id: "tsm4", time: now - day * 10, color: "#999999", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"]}, - {id: "tsm5", time: now - day * 14, color: "purple", label: "S", tooltip: ["Split: 2/1", "Date: " + new Date((now - day * 14) * 1000).toDateString()]}, - {id: "tsm6", time: now - day * 18, color: "black", label: "E", tooltip: ["Earnings: $5.40", "Estimate: $5.00"]}, + {id: "tsm4", time: now - day * 15, color: "#999999", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"]}, + {id: "tsm7", time: now - day * 30, color: "red", label: "E", tooltip: ["Earnings: $5.40", "Estimate: $5.00"]}, ]; response.writeHead(200, defaultResponseHeader); From 16726cb1a51e6c23c96c206202d65484e7d4def9 Mon Sep 17 00:00:00 2001 From: Eugene Date: Tue, 13 Oct 2015 17:26:48 +0300 Subject: [PATCH 07/59] Added time request --- yahoo.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/yahoo.js b/yahoo.js index 5e72356..0c0b940 100644 --- a/yahoo.js +++ b/yahoo.js @@ -146,6 +146,7 @@ RequestProcessor = function(action, query, response) { supports_group_request: false, supports_marks: true, supports_timescale_marks: true, + supports_time: true, exchanges: [ {value: "", name: "All Exchanges", desc: ""}, {value: "XETRA", name: "XETRA", desc: "XETRA"}, @@ -189,6 +190,13 @@ RequestProcessor = function(action, query, response) { response.end(); } + this.sendTime = function(response) { + var now = new Date(); + response.writeHead(200, defaultResponseHeader); + response.write(Math.floor(now / 1000) + ''); + response.end(); + }; + this.sendTimescaleMarks = function(response) { var now = new Date(); now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; @@ -386,6 +394,9 @@ RequestProcessor = function(action, query, response) { else if (action == "/marks") { this.sendMarks(response); } + else if (action == "/time") { + this.sendTime(response); + } else if (action == "/timescale_marks") { this.sendTimescaleMarks(response); } From 00222f835cbcd0364b4627f0a1151dc8b4394595 Mon Sep 17 00:00:00 2001 From: Nipheris Date: Fri, 16 Oct 2015 16:31:05 +0300 Subject: [PATCH 08/59] Support for symbol search by description --- symbols_database.js | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/symbols_database.js b/symbols_database.js index 51f094a..858e1f6 100644 --- a/symbols_database.js +++ b/symbols_database.js @@ -171,10 +171,11 @@ function searchResultFromDatabaseItem(item) { } -exports.search = function (searchText, type, exchange, maxRecords) { +exports.search = function (searchString, type, exchange, maxRecords) { var MAX_SEARCH_RESULTS = !!maxRecords ? maxRecords : 50; var results = []; - var queryIsEmpty = !searchText || searchText.length == 0; + var queryIsEmpty = !searchString || searchString.length == 0; + var searchStringUpperCase = searchString.toUpperCase(); for (var i = 0; i < symbols.length; ++i) { var item = symbols[i]; @@ -184,7 +185,9 @@ exports.search = function (searchText, type, exchange, maxRecords) { if (exchange && exchange.length > 0 && item.exchange != exchange) { continue; } - if (queryIsEmpty || item.name.indexOf(searchText) == 0) { + if (queryIsEmpty || + item.name.toUpperCase().indexOf(searchStringUpperCase) == 0 || + item.description.toUpperCase().indexOf(searchStringUpperCase) >= 0) { results.push(searchResultFromDatabaseItem(item)); } if (results.length >= MAX_SEARCH_RESULTS) { From c52339033491d7db05478988b41b1a88de7f7d1c Mon Sep 17 00:00:00 2001 From: Stanislav Makarov Date: Wed, 21 Oct 2015 19:44:19 +0300 Subject: [PATCH 09/59] Weighted symbol search --- symbols_database.js | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/symbols_database.js b/symbols_database.js index 858e1f6..5d38845 100644 --- a/symbols_database.js +++ b/symbols_database.js @@ -173,28 +173,45 @@ function searchResultFromDatabaseItem(item) { exports.search = function (searchString, type, exchange, maxRecords) { var MAX_SEARCH_RESULTS = !!maxRecords ? maxRecords : 50; - var results = []; + var results = []; // array of WeightedItem { item, weight } var queryIsEmpty = !searchString || searchString.length == 0; var searchStringUpperCase = searchString.toUpperCase(); for (var i = 0; i < symbols.length; ++i) { var item = symbols[i]; + if (type && type.length > 0 && item.type != type) { continue; } if (exchange && exchange.length > 0 && item.exchange != exchange) { continue; } - if (queryIsEmpty || - item.name.toUpperCase().indexOf(searchStringUpperCase) == 0 || - item.description.toUpperCase().indexOf(searchStringUpperCase) >= 0) { - results.push(searchResultFromDatabaseItem(item)); + + var positionInName = item.name.toUpperCase().indexOf(searchStringUpperCase); + var positionInDescription = item.description.toUpperCase().indexOf(searchStringUpperCase); + + if (queryIsEmpty || positionInName >= 0 || positionInDescription >= 0) { + var found = false; + for (var resultIndex = 0; resultIndex < results.length; resultIndex++) { + if (results[resultIndex].item == item) { + found = true; + break; + } + } + if (!found) { + var weight = positionInName >= 0 ? positionInName : 8000 + positionInDescription; + results.push({ item: item, weight: weight }); + } } + if (results.length >= MAX_SEARCH_RESULTS) { break; } } - return results; + + return results + .sort(function (weightedItem1, weightedItem2) { return weightedItem1.weight - weightedItem2.weight; }) + .map(function (weightedItem) { return searchResultFromDatabaseItem(weightedItem.item); }); } From a1bd5e92a93b7015c19d8c730c0443e8caf52309 Mon Sep 17 00:00:00 2001 From: Stanislav Makarov Date: Fri, 23 Oct 2015 18:01:05 +0300 Subject: [PATCH 10/59] Fixed incorrect truncation of search results Limitations should be applied AFTER ordering results by weight, not before --- symbols_database.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/symbols_database.js b/symbols_database.js index 5d38845..05c3661 100644 --- a/symbols_database.js +++ b/symbols_database.js @@ -203,15 +203,12 @@ exports.search = function (searchString, type, exchange, maxRecords) { results.push({ item: item, weight: weight }); } } - - if (results.length >= MAX_SEARCH_RESULTS) { - break; - } } return results .sort(function (weightedItem1, weightedItem2) { return weightedItem1.weight - weightedItem2.weight; }) - .map(function (weightedItem) { return searchResultFromDatabaseItem(weightedItem.item); }); + .map(function (weightedItem) { return searchResultFromDatabaseItem(weightedItem.item); }) + .slice(0, Math.min(results.length, MAX_SEARCH_RESULTS)); } From fc2b18e0e547ab1db02ae86e3f49201bad24c261 Mon Sep 17 00:00:00 2001 From: Eugene Date: Fri, 20 Nov 2015 16:30:31 +0300 Subject: [PATCH 11/59] send content-length --- yahoo.js | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/yahoo.js b/yahoo.js index 0c0b940..3078cf4 100644 --- a/yahoo.js +++ b/yahoo.js @@ -11,7 +11,12 @@ var http = require("http"), symbolsDatabase = require("./symbols_database"); var datafeedHost = "chartapi.finance.yahoo.com"; -var defaultResponseHeader = {"Content-Type": "text/plain", 'Access-Control-Allow-Origin': '*'}; + +function createDefaultHeader() { + return {"Content-Type": "text/plain", 'Access-Control-Allow-Origin': '*'}; +} + +var defaultResponseHeader = createDefaultHeader(); function httpGet(path, callback) @@ -331,8 +336,11 @@ RequestProcessor = function(action, query, response) { var that = this; httpGet(address, function(result) { - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(convertYahooHistoryToUDFFormat(result))); + var content = JSON.stringify(convertYahooHistoryToUDFFormat(result)); + var header = createDefaultHeader(); + header["Content-Length"] = content.length; + response.writeHead(200, header); + response.write(content); response.end(); }); } From ae9f1eeba1dd553c3c4be0003669080ad1d0b7bc Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Tue, 5 Jul 2016 13:24:37 +0300 Subject: [PATCH 12/59] fixed crashes on yahoo timeouts --- yahoo.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/yahoo.js b/yahoo.js index 3078cf4..4a0b7df 100644 --- a/yahoo.js +++ b/yahoo.js @@ -374,6 +374,12 @@ RequestProcessor = function(action, query, response) { }); res.on('end', function () { + if (res.statusCode !== 200) { + response.writeHead(204, defaultResponseHeader); + console.error('Wrong response: ' + result); + response.end(); + return; + } response.writeHead(200, defaultResponseHeader); response.write(JSON.stringify(convertYahooQuotesToUDFFormat( tickersMap, JSON.parse(result)))); From ddb0033d701666e6aea7984139ce6c5a2b666b2c Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Wed, 20 Jul 2016 16:57:00 +0300 Subject: [PATCH 13/59] partial yahoo requests --- yahoo.js | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/yahoo.js b/yahoo.js index 4a0b7df..496d7a9 100644 --- a/yahoo.js +++ b/yahoo.js @@ -34,6 +34,11 @@ function httpGet(path, callback) }); response.on('end', function () { + if (response.statusCode !== 200) { + callback(''); + return; + } + callback(result) }); } @@ -77,7 +82,7 @@ function convertYahooHistoryToUDFFormat(data) { var items = lines[i].split(","); var time = parseDate(items[0]) / 1000; - + result.t.push(time); result.o.push(parseFloat(items[1])); result.h.push(parseFloat(items[2])); @@ -85,6 +90,10 @@ function convertYahooHistoryToUDFFormat(data) { result.c.push(parseFloat(items[4])); result.v.push(parseFloat(items[5])); } + + if (result.t.length === 0) { + result.s = "no_data"; + } return result; } @@ -305,7 +314,7 @@ RequestProcessor = function(action, query, response) { } - this.sendSymbolHistory = function(symbol, startDateTimestamp, resolution, response) { + this.sendSymbolHistory = function(symbol, startDateTimestamp, endDateTimestamp, resolution, response) { var symbolInfo = symbolsDatabase.symbolInfo(symbol); @@ -319,6 +328,19 @@ RequestProcessor = function(action, query, response) { var year = requestLeftDate.getFullYear(); var month = requestLeftDate.getMonth(); var day = requestLeftDate.getDate(); + + var endtext = ''; + + if (endDateTimestamp) { + var requestRightDate = new Date(endDateTimestamp * 1000); + var endyear = requestRightDate.getFullYear(); + var endmonth = requestRightDate.getMonth(); + var endday = requestRightDate.getDate(); + + endtext = '&d=' + endmonth + + '&e=' + endday + + '&f=' + endyear; + } if (resolution != "d" && resolution != "w" && resolution != "m") { throw "Unsupported resolution: " + resolution; @@ -327,7 +349,7 @@ RequestProcessor = function(action, query, response) { var address = "ichart.finance.yahoo.com/table.csv?s=" + symbolInfo.name + "&a=" + month + "&b=" + day + - "&c=" + year + + "&c=" + year + endtext + "&g=" + resolution + "&ignore=.csv"; @@ -400,7 +422,7 @@ RequestProcessor = function(action, query, response) { this.sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); } else if (action == "/history") { - this.sendSymbolHistory(query["symbol"], query["from"], query["resolution"].toLowerCase(), response); + this.sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response); } else if (action == "/quotes") { this.sendQuotes(query["symbols"], response); From 50dd738d403513236c0f61581fa8a084ec9ee3cd Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 21 Jul 2016 11:37:04 +0300 Subject: [PATCH 14/59] return good response --- yahoo.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/yahoo.js b/yahoo.js index 496d7a9..c5e3b3a 100644 --- a/yahoo.js +++ b/yahoo.js @@ -397,8 +397,8 @@ RequestProcessor = function(action, query, response) { res.on('end', function () { if (res.statusCode !== 200) { - response.writeHead(204, defaultResponseHeader); - console.error('Wrong response: ' + result); + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify({ s: 'error', errmsg: 'Yahoo fails' })); response.end(); return; } From d85157dabb0297c2f8bea8d4b97782396eaf38f2 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 12 Jan 2017 16:12:58 +0300 Subject: [PATCH 15/59] add news proxy, because google api is deprecated --- yahoo.js | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/yahoo.js b/yahoo.js index c5e3b3a..a34bae9 100644 --- a/yahoo.js +++ b/yahoo.js @@ -7,6 +7,7 @@ */ var http = require("http"), + https = require("https"), url = require("url"), symbolsDatabase = require("./symbols_database"); @@ -142,6 +143,28 @@ function convertYahooQuotesToUDFFormat(tickersMap, data) { return result; } +function proxyRequest(controller, options, response) { + controller.request(options, function(res) { + var result = ''; + + res.on('data', function (chunk) { + result += chunk; + }); + + res.on('end', function () { + if (res.statusCode !== 200) { + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify({ s: 'error', errmsg: 'Failed to get news' })); + response.end(); + return; + } + response.writeHead(200, defaultResponseHeader); + response.write(result); + response.end(); + }); + }).end(); +} + RequestProcessor = function(action, query, response) { this.sendError = function(error, response) { @@ -409,6 +432,24 @@ RequestProcessor = function(action, query, response) { }); }).end(); } + + this.sendNews = function(symbol, response) { + var options = { + host: "feeds.finance.yahoo.com", + path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US", + }; + + proxyRequest(https, options, response); + } + + this.sendFuturesmag = function(response) { + var options = { + host: "www.futuresmag.com", + path: "/rss/all", + }; + + proxyRequest(http, options, response); + } try { @@ -436,6 +477,12 @@ RequestProcessor = function(action, query, response) { else if (action == "/timescale_marks") { this.sendTimescaleMarks(response); } + else if (action == "/news") { + this.sendNews(query["symbol"], response); + } + else if (action == "/futuresmag") { + this.sendFuturesmag(response); + } } catch (error) { this.sendError(error, response) From eb73a479336e00689aabe30e6ef435d08cb6316f Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Fri, 31 Mar 2017 15:54:23 +0300 Subject: [PATCH 16/59] a way to set port from env --- yahoo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yahoo.js b/yahoo.js index a34bae9..c26ac5c 100644 --- a/yahoo.js +++ b/yahoo.js @@ -496,7 +496,7 @@ RequestProcessor = function(action, query, response) { // /search?query=B&limit=10 // /history?symbol=C&from=DATE&resolution=E -var firstPort = 8888; +var firstPort = process.env.YAHOO_PORT || 8888; function getFreePort(callback) { var port = firstPort; firstPort++; From 8757ff9472100b5fbd79c3532caddad7997fc80f Mon Sep 17 00:00:00 2001 From: Eugene Timokhov Date: Mon, 17 Apr 2017 13:42:54 +0300 Subject: [PATCH 17/59] Used https requests to yahoo to avoid 301 --- yahoo.js | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/yahoo.js b/yahoo.js index c26ac5c..113b8aa 100644 --- a/yahoo.js +++ b/yahoo.js @@ -44,21 +44,21 @@ function httpGet(path, callback) }); } - var req = http.request(options, onDataCallback); - + var req = https.request(options, onDataCallback); + req.on('socket', function (socket) { - socket.setTimeout(5000); + socket.setTimeout(5000); socket.on('timeout', function() { console.log('timeout'); req.abort(); }); }); - + req.on('error', function(e) { console.log('Problem with request: ' + e.message); callback(''); }); - + req.end(); } @@ -83,7 +83,7 @@ function convertYahooHistoryToUDFFormat(data) { var items = lines[i].split(","); var time = parseDate(items[0]) / 1000; - + result.t.push(time); result.o.push(parseFloat(items[1])); result.h.push(parseFloat(items[2])); @@ -91,7 +91,7 @@ function convertYahooHistoryToUDFFormat(data) { result.c.push(parseFloat(items[4])); result.v.push(parseFloat(items[5])); } - + if (result.t.length === 0) { result.s = "no_data"; } @@ -226,24 +226,24 @@ RequestProcessor = function(action, query, response) { response.write(JSON.stringify(marks)); response.end(); } - + this.sendTime = function(response) { var now = new Date(); response.writeHead(200, defaultResponseHeader); response.write(Math.floor(now / 1000) + ''); response.end(); }; - + this.sendTimescaleMarks = function(response) { var now = new Date(); now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; var day = 60 * 60 * 24; - + var marks = [ {id: "tsm1", time: now - day * 0, color: "red", label: "A", tooltip: ""}, {id: "tsm2", time: now - day * 4, color: "blue", label: "D", tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()]}, {id: "tsm3", time: now - day * 7, color: "green", label: "D", tooltip: ["Dividends: $3.46", "Date: " + new Date((now - day * 7) * 1000).toDateString()]}, - {id: "tsm4", time: now - day * 15, color: "#999999", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"]}, + {id: "tsm4", time: now - day * 15, color: "#999999", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"]}, {id: "tsm7", time: now - day * 30, color: "red", label: "E", tooltip: ["Earnings: $5.40", "Estimate: $5.00"]}, ]; @@ -351,15 +351,15 @@ RequestProcessor = function(action, query, response) { var year = requestLeftDate.getFullYear(); var month = requestLeftDate.getMonth(); var day = requestLeftDate.getDate(); - + var endtext = ''; - - if (endDateTimestamp) { - var requestRightDate = new Date(endDateTimestamp * 1000); + + if (endDateTimestamp) { + var requestRightDate = new Date(endDateTimestamp * 1000); var endyear = requestRightDate.getFullYear(); var endmonth = requestRightDate.getMonth(); var endday = requestRightDate.getDate(); - + endtext = '&d=' + endmonth + '&e=' + endday + '&f=' + endyear; @@ -391,7 +391,7 @@ RequestProcessor = function(action, query, response) { } this.sendQuotes = function(tickersString, response) { - var tickersMap = {}; // maps YQL symbol to ticker + var tickersMap = {}; // maps YQL symbol to ticker var tickers = tickersString.split(","); [].concat(tickers).forEach(function(ticker) { @@ -432,7 +432,7 @@ RequestProcessor = function(action, query, response) { }); }).end(); } - + this.sendNews = function(symbol, response) { var options = { host: "feeds.finance.yahoo.com", @@ -441,7 +441,7 @@ RequestProcessor = function(action, query, response) { proxyRequest(https, options, response); } - + this.sendFuturesmag = function(response) { var options = { host: "www.futuresmag.com", From 08a685e75f561e96adfba28eac51494d24abb901 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 18 May 2017 16:27:44 +0300 Subject: [PATCH 18/59] quandl source --- yahoo.js | 175 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 139 insertions(+), 36 deletions(-) diff --git a/yahoo.js b/yahoo.js index 113b8aa..2a0e320 100644 --- a/yahoo.js +++ b/yahoo.js @@ -12,6 +12,8 @@ var http = require("http"), symbolsDatabase = require("./symbols_database"); var datafeedHost = "chartapi.finance.yahoo.com"; +var lastHistoryErrorTime = null; +var errorSwitchingTime = 60 * 60 * 1000; // switch to Quandl for 1 hour function createDefaultHeader() { return {"Content-Type": "text/plain", 'Access-Control-Allow-Origin': '*'}; @@ -19,8 +21,7 @@ function createDefaultHeader() { var defaultResponseHeader = createDefaultHeader(); - -function httpGet(path, callback) +function httpGet(datafeedHost, path, callback, failedCallback) { var options = { host: datafeedHost, @@ -56,7 +57,7 @@ function httpGet(path, callback) req.on('error', function(e) { console.log('Problem with request: ' + e.message); - callback(''); + failedCallback ? failedCallback(e) : callback(''); }); req.end(); @@ -99,6 +100,49 @@ function convertYahooHistoryToUDFFormat(data) { return result; } +function convertQuandlHistoryToUDFFormat(data) { + function parseDate(input) { + var parts = input.split('-'); + return Date.UTC(parts[0], parts[1]-1, parts[2]); + } + + function columnIndices(columns) { + var indices = {}; + for (var i = 0; i < columns.length; i++) { + indices[columns[i].name] = i; + } + + return indices; + } + + var result = { + t: [], c: [], o: [], h: [], l: [], v: [], + s: "ok" + }; + + try { + var json = JSON.parse(data); + var datatable = json.datatable; + var data = datatable.data; + var columns = datatable.columns; + var idx = columnIndices(columns); + + data.forEach(function(row) { + result.t.push(parseDate(row[idx.date]) / 1000); + result.o.push(row[idx.open]); + result.h.push(row[idx.high]); + result.l.push(row[idx.low]); + result.c.push(row[idx.close]); + result.v.push(row[idx.volume]); + }); + + } catch(error) { + console.error(error); + } + + return result; +} + function convertYahooQuotesToUDFFormat(tickersMap, data) { if (!data.query || !data.query.results) { var errmsg = "ERROR: empty quotes response: " + JSON.stringify(data); @@ -285,13 +329,39 @@ RequestProcessor = function(action, query, response) { if (symbolInfo == null) { throw "unknown_symbol " + symbolName; } + + var info = { + "name": symbolInfo.name, + "exchange-traded": symbolInfo.exchange, + "exchange-listed": symbolInfo.exchange, + "timezone": "America/New_York", + "minmov": 1, + "minmov2": 0, + "pointvalue": 1, + "session": "0930-1630", + "has_intraday": false, + "has_no_volume": symbolInfo.type != "stock", + "description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name, + "type": symbolInfo.type, + "supported_resolutions" : ["D","2D","3D","W","3W","M","6M"], + "pricescale": 100, + "ticker": symbolInfo.name.toUpperCase(), + }; + + if (lastHistoryErrorTime && Date.now() - lastHistoryErrorTime < errorSwitchingTime) { + // return default response if we have problems with Yahoo + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(info)); + response.end(); + return; + } var address = "/instrument/1.0/" + encodeURIComponent(symbolInfo.name) + "/chartdata;type=quote;/json"; var that = this; console.log(datafeedHost + address); - httpGet(address, function(result) { + httpGet(datafeedHost, address, function(result) { _pendingRequestType = "meta"; try { @@ -303,41 +373,69 @@ RequestProcessor = function(action, query, response) { that.sendError("invalid symbol", response); return; } - - var lastPrice = _lastYahooResponse["previous_close"] + ""; - - // BEWARE: this `pricescale` parameter computation algorithm is wrong and works - // for symbols with 10-based minimal movement value only - var pricescale = lastPrice.indexOf('.') > 0 - ? Math.pow(10, lastPrice.split('.')[1].length) - : 10; - - var info = { - "name": symbolInfo.name, - "exchange-traded": symbolInfo.exchange, - "exchange-listed": symbolInfo.exchange, - "timezone": "America/New_York", - "minmov": 1, - "minmov2": 0, - "pricescale": pricescale, - "pointvalue": 1, - "session": "0930-1630", - "has_intraday": false, - "has_no_volume": symbolInfo.type != "stock", - "ticker": _lastYahooResponse["ticker"].toUpperCase(), - "description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name, - "type": symbolInfo.type, - "supported_resolutions" : ["D","2D","3D","W","3W","M","6M"] - }; - + + try { + var lastPrice = _lastYahooResponse["previous_close"] + ""; + + // BEWARE: this `pricescale` parameter computation algorithm is wrong and works + // for symbols with 10-based minimal movement value only + var pricescale = lastPrice.indexOf('.') > 0 + ? Math.pow(10, lastPrice.split('.')[1].length) + : 10; + + Object.assign(info, { + "pricescale": pricescale, + "ticker": _lastYahooResponse["ticker"].toUpperCase(), + }); + } catch(error) { + console.error(error); + } + + response.writeHead(200, defaultResponseHeader); response.write(JSON.stringify(info)); response.end(); }); } + function requestHistoryFromQuandl(symbol, startDateTimestamp, endDateTimestamp, response) { + function dateToYMD(date) { + var obj = new Date(date * 1000); + var year = obj.getFullYear(); + var month = obj.getMonth(); + var day = obj.getDate(); + return year + "-" + month + "-" + day; + } + var address = "/api/v3/datatables/WIKI/PRICES.json" + + "?api_key=" + process.env.QUANDL_API_KEY + // you should create a free account on quandl.com to get this key + "&ticker=" + symbol + + "&date.gte=" + dateToYMD(startDateTimestamp) + + "&date.lte=" + dateToYMD(endDateTimestamp); + + console.log("Sending request to quandl for symbol " + symbol + ". url=" + address); + + httpGet("www.quandl.com", address, function(result) { + if (response.finished) { + // we can be here if error happened on socket disconnect + return; + } + inCallback = true; + var content = JSON.stringify(convertQuandlHistoryToUDFFormat(result)); + var header = createDefaultHeader(); + header["Content-Length"] = content.length; + response.writeHead(200, header); + response.write(content, null, function() { + response.end(); + }); + }); + + }; - this.sendSymbolHistory = function(symbol, startDateTimestamp, endDateTimestamp, resolution, response) { + this.sendSymbolHistory = function(symbol, startDateTimestamp, endDateTimestamp, resolution, response) { + if (lastHistoryErrorTime && Date.now() - lastHistoryErrorTime < errorSwitchingTime) { + requestHistoryFromQuandl(symbol, startDateTimestamp, endDateTimestamp, response); + return; + } var symbolInfo = symbolsDatabase.symbolInfo(symbol); @@ -380,15 +478,20 @@ RequestProcessor = function(action, query, response) { var that = this; - httpGet(address, function(result) { + httpGet(datafeedHost, address, function(result) { var content = JSON.stringify(convertYahooHistoryToUDFFormat(result)); var header = createDefaultHeader(); header["Content-Length"] = content.length; response.writeHead(200, header); - response.write(content); - response.end(); + response.write(content, null, function() { + response.end(); + }); + }, function(error) { + // try another feed + requestHistoryFromQuandl(symbol, startDateTimestamp, endDateTimestamp, response); + lastHistoryErrorTime = Date.now(); }); - } + } this.sendQuotes = function(tickersString, response) { var tickersMap = {}; // maps YQL symbol to ticker From 60a7d6185673f057c2a33d0458fcbc8cb08cb303 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Fri, 19 May 2017 12:15:25 +0300 Subject: [PATCH 19/59] cache quandl results --- yahoo.js | 46 ++++++++++++++++++++++++++++++++++------------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/yahoo.js b/yahoo.js index 2a0e320..2512da0 100644 --- a/yahoo.js +++ b/yahoo.js @@ -14,6 +14,12 @@ var http = require("http"), var datafeedHost = "chartapi.finance.yahoo.com"; var lastHistoryErrorTime = null; var errorSwitchingTime = 60 * 60 * 1000; // switch to Quandl for 1 hour +var quandlCache = {}; + +var quandlCacheCleanupTime = 3 * 60 * 60 * 100; // 3 hours +setInterval(function() { + quandlCache = {}; +}, quandlCacheCleanupTime); function createDefaultHeader() { return {"Content-Type": "text/plain", 'Access-Control-Allow-Origin': '*'}; @@ -37,7 +43,7 @@ function httpGet(datafeedHost, path, callback, failedCallback) response.on('end', function () { if (response.statusCode !== 200) { - callback(''); + failedCallback ? failedCallback(response.statusCode) : callback(''); return; } @@ -406,11 +412,32 @@ RequestProcessor = function(action, query, response) { var day = obj.getDate(); return year + "-" + month + "-" + day; } + + function sendResult(content) { + var header = createDefaultHeader(); + header["Content-Length"] = content.length; + response.writeHead(200, header); + response.write(content, null, function() { + response.end(); + }); + } + + var from = dateToYMD(startDateTimestamp); + var to = dateToYMD(endDateTimestamp); + + var key = symbol + "|" + from + "|" + to; + + if (quandlCache[key]) { + console.log("Return QUANDL result from cache: " + key); + sendResult(quandlCache[key]); + return; + } + var address = "/api/v3/datatables/WIKI/PRICES.json" + "?api_key=" + process.env.QUANDL_API_KEY + // you should create a free account on quandl.com to get this key "&ticker=" + symbol + - "&date.gte=" + dateToYMD(startDateTimestamp) + - "&date.lte=" + dateToYMD(endDateTimestamp); + "&date.gte=" + from + + "&date.lte=" + to; console.log("Sending request to quandl for symbol " + symbol + ". url=" + address); @@ -418,15 +445,10 @@ RequestProcessor = function(action, query, response) { if (response.finished) { // we can be here if error happened on socket disconnect return; - } - inCallback = true; - var content = JSON.stringify(convertQuandlHistoryToUDFFormat(result)); - var header = createDefaultHeader(); - header["Content-Length"] = content.length; - response.writeHead(200, header); - response.write(content, null, function() { - response.end(); - }); + } + var content = JSON.stringify(convertQuandlHistoryToUDFFormat(result)); + quandlCache[key] = content; + sendResult(content); }); }; From eab0a08bc937297fcbc2fedb183aa5c87c99739d Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 6 Jul 2017 19:43:23 +0300 Subject: [PATCH 20/59] change first date --- yahoo.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/yahoo.js b/yahoo.js index 2512da0..0ba19b5 100644 --- a/yahoo.js +++ b/yahoo.js @@ -422,7 +422,7 @@ RequestProcessor = function(action, query, response) { }); } - var from = dateToYMD(startDateTimestamp); + var from = '1970-01-01'; // dateToYMD(startDateTimestamp); always return all data to reduce number of requests to quandl var to = dateToYMD(endDateTimestamp); var key = symbol + "|" + from + "|" + to; From acd4651d4ac31a2aea4b0f489ce9653c418f962f Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 20 Jul 2017 12:59:41 +0300 Subject: [PATCH 21/59] remove yahoo history requests --- README.md | 9 +- symbols_database.js | 279 +++++++++++++++++++------------------------- yahoo.js | 274 +++++++++---------------------------------- 3 files changed, 186 insertions(+), 376 deletions(-) diff --git a/README.md b/README.md index 049a07e..22e14a9 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,10 @@ -UDF-compatible Yahoo datafeed +UDF-compatible Quandl/Yahoo datafeed ============== This repository contains a sample implementation of server-side UDF-compatible data source. -Use nodejs to launch yahoo.js \ No newline at end of file + +Register for free at www.quandl.com to get your free API key. + +Set QUANDL_API_KEY environment variable to your Quandl key before starting the feed. + +Use NodeJS to launch yahoo.js \ No newline at end of file diff --git a/symbols_database.js b/symbols_database.js index 05c3661..507112c 100644 --- a/symbols_database.js +++ b/symbols_database.js @@ -1,163 +1,128 @@ /* This file is a node.js module intended for use in different UDF datafeeds. */ -// This list should contain all the symbosl available through your datafeed. -// The current version is extremely incomplete (as it's just a sample): Yahoo has much more of them. - - -var symbols = [ -{ name: "^GDAXI", description:"DAX", exchange:"XETRA", type:"index" }, -{ name: "^NSEI", description:"CNX NIFTY", exchange:"NSE", type:"index" }, -{ name: "A", description:"Agilent Technologies Inc.", exchange:"NYSE", type:"stock" }, -{ name: "AA", description:"Alcoa Inc.", exchange:"NYSE", type:"stock" }, -{ name: "AAL", description:"American Airlines Group Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AAPL", description:"Apple Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ABB", description:"ABB Ltd.", exchange:"NYSE", type:"stock" }, -{ name: "ABBV", description:"AbbVie Inc.", exchange:"NYSE", type:"stock" }, -{ name: "ABT", description:"Abbott Laboratories", exchange:"NYSE", type:"stock" }, -{ name: "ABX", description:"Barrick Gold Corporation", exchange:"NYSE", type:"stock" }, -{ name: "ACHN", description:"Achillion Pharmaceuticals, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ACI", description:"Arch Coal Inc.", exchange:"NYSE", type:"stock" }, -{ name: "ACN", description:"Accenture plc", exchange:"NYSE", type:"stock" }, -{ name: "ACT", description:"Actavis plc", exchange:"NYSE", type:"stock" }, -{ name: "ADBE", description:"Adobe Systems Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ADSK", description:"Autodesk, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AEO", description:"American Eagle Outfitters, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "AGNC", description:"American Capital Agency Corp.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AIG", description:"American International Group, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "AKAM", description:"Akamai Technologies, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ALU", description:"Alcatel-Lucent", exchange:"NYSE", type:"stock" }, -{ name: "ALXN", description:"Alexion Pharmaceuticals, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AMAT", description:"Applied Materials, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AMD", description:"Advanced Micro Devices, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "AMGN", description:"Amgen Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AMZN", description:"Amazon.com Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ANF", description:"Abercrombie & Fitch Co.", exchange:"NYSE", type:"stock" }, -{ name: "ANR", description:"Alpha Natural Resources, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "APA", description:"Apache Corp.", exchange:"NYSE", type:"stock" }, -{ name: "APC", description:"Anadarko Petroleum Corporation", exchange:"NYSE", type:"stock" }, -{ name: "ARC", description:"ARC Document Solutions, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "ARIA", description:"Ariad Pharmaceuticals Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ARNA", description:"Arena Pharmaceuticals, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ARR", description:"ARMOUR Residential REIT, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "AUXL", description:"Auxilium Pharmaceuticals Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AVGO", description:"Avago Technologies Limited", exchange:"NasdaqNM", type:"stock" }, -{ name: "AVNR", description:"Avanir Pharmaceuticals, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AWAY", description:"HomeAway, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "AXP", description:"American Express Company", exchange:"NYSE", type:"stock" }, -{ name: "AZO", description:"AutoZone, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "BA", description:"The Boeing Company", exchange:"NYSE", type:"stock" }, -{ name: "BAC", description:"Bank of America Corporation", exchange:"NYSE", type:"stock" }, -{ name: "BAX", description:"Baxter International Inc.", exchange:"NYSE", type:"stock" }, -{ name: "BBBY", description:"Bed Bath & Beyond Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "BBRY", description:"BlackBerry Limited", exchange:"NasdaqNM", type:"stock" }, -{ name: "BBT", description:"BB&T Corporation", exchange:"NYSE", type:"stock" }, -{ name: "BBY", description:"Best Buy Co., Inc.", exchange:"NYSE", type:"stock" }, -{ name: "BIDU", description:"Baidu, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "BIIB", description:"Biogen Idec Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "BK", description:"The Bank of New York Mellon Corporation", exchange:"NYSE", type:"stock" }, -{ name: "BLK", description:"BlackRock, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "BMO", description:"Bank of Montreal", exchange:"NYSE", type:"stock" }, -{ name: "BMY", description:"Bristol-Myers Squibb Company", exchange:"NYSE", type:"stock" }, -{ name: "BP", description:"BP plc", exchange:"NYSE", type:"stock" }, -{ name: "BRCD", description:"Brocade Communications Systems, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "BRCM", description:"Broadcom Corp.", exchange:"NasdaqNM", type:"stock" }, -{ name: "BRK-B", description:"Berkshire Hathaway Inc.", exchange:"NYSE", type:"stock" }, -{ name: "BTU", description:"Peabody Energy Corp.", exchange:"NYSE", type:"stock" }, -{ name: "BX", description:"The Blackstone Group L.P.", exchange:"NYSE", type:"stock" }, -{ name: "C", description:"Citigroup Inc.", exchange:"NYSE", type:"stock" }, -{ name: "CHK", description:"Chesapeake Energy Corporation", exchange:"NYSE", type:"stock" }, -{ name: "CII.TA", description:"", exchange:"Tel Aviv", type:"stock" }, -{ name: "CNP", description:"CenterPoint Energy, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "COLE", description:"Cole Real Estate Investments, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "CSCO", description:"Cisco Systems, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "D", description:"Dominion Resources, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DAL", description:"Delta Air Lines Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DANG", description:"E-Commerce China Dangdang Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DBD", description:"Diebold, Incorporated", exchange:"NYSE", type:"stock" }, -{ name: "DCM", description:"NTT DOCOMO, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DD", description:"E. I. du Pont de Nemours and Company", exchange:"NYSE", type:"stock" }, -{ name: "DDD", description:"3D Systems Corp.", exchange:"NYSE", type:"stock" }, -{ name: "DE", description:"Deere & Company", exchange:"NYSE", type:"stock" }, -{ name: "DE.AS", description:"", exchange:"Amsterdam", type:"stock" }, -{ name: "DECK", description:"Deckers Outdoor Corp.", exchange:"NYSE", type:"stock" }, -{ name: "DEI", description:"Douglas Emmett Inc", exchange:"NYSE", type:"stock" }, -{ name: "DHI", description:"DR Horton Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DIS", description:"The Walt Disney Company", exchange:"NYSE", type:"stock" }, -{ name: "DLTR", description:"Dollar Tree, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "DNDN", description:"Dendreon Corp.", exchange:"NasdaqNM", type:"stock" }, -{ name: "DO", description:"Diamond Offshore Drilling, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DOV", description:"Dover Corporation", exchange:"NYSE", type:"stock" }, -{ name: "DOW", description:"The Dow Chemical Company", exchange:"NYSE", type:"stock" }, -{ name: "DPM", description:"DCP Midstream Partners LP", exchange:"NYSE", type:"stock" }, -{ name: "DRI", description:"Darden Restaurants, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DRX.IR", description:"", exchange:"Irish", type:"stock" }, -{ name: "DRYS", description:"DryShips, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "DV", description:"DeVry Education Group Inc.", exchange:"NYSE", type:"stock" }, -{ name: "DVN", description:"Devon Energy Corporation", exchange:"NYSE", type:"stock" }, -{ name: "EA", description:"Electronic Arts Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "EBAY", description:"eBay Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "EBIX", description:"Ebix Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ECYT", description:"Endocyte, Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "ED", description:"Consolidated Edison, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "EMC", description:"EMC Corporation", exchange:"NYSE", type:"stock" }, -{ name: "ENT", description:"Global Eagle Entertainment Inc.", exchange:"NCM", type:"stock" }, -{ name: "ESI", description:"ITT Educational Services Inc.", exchange:"NYSE", type:"stock" }, -{ name: "ESRX", description:"Express Scripts Holding Company", exchange:"NasdaqNM", type:"stock" }, -{ name: "ETFC", description:"E*TRADE Financial Corporation", exchange:"NasdaqNM", type:"stock" }, -{ name: "ETP", description:"Energy Transfer Partners, L.P.", exchange:"NYSE", type:"stock" }, -{ name: "EXC", description:"Exelon Corporation", exchange:"NYSE", type:"stock" }, -{ name: "EXPE", description:"Expedia Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "EZCH", description:"EZchip Semiconductor Ltd.", exchange:"NasdaqNM", type:"stock" }, -{ name: "F", description:"Ford Motor Co.", exchange:"NYSE", type:"stock" }, -{ name: "FCEL", description:"FuelCell Energy Inc.", exchange:"NGM", type:"stock" }, -{ name: "FRE.AX", description:"Freshtel Holdings Limited", exchange:"ASX", type:"stock" }, -{ name: "GALE", description:"Galena Biopharma, Inc.", exchange:"NCM", type:"stock" }, -{ name: "GD", description:"General Dynamics Corp.", exchange:"NYSE", type:"stock" }, -{ name: "GE", description:"General Electric Company", exchange:"NYSE", type:"stock" }, -{ name: "GTAT", description:"GT Advanced Technologies Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "HD", description:"The Home Depot, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "IBM", description:"International Business Machines Corporation", exchange:"NYSE", type:"stock" }, -{ name: "INTC", description:"Intel Corporation", exchange:"NasdaqNM", type:"stock" }, -{ name: "JPM", description:"JPMorgan Chase & Co.", exchange:"NYSE", type:"stock" }, -{ name: "KERX", description:"Keryx Biopharmaceuticals Inc.", exchange:"NCM", type:"stock" }, -{ name: "KMP", description:"Kinder Morgan Energy Partners, L.P.", exchange:"NYSE", type:"stock" }, -{ name: "KO", description:"The Coca-Cola Company", exchange:"NYSE", type:"stock" }, -{ name: "LINE", description:"Linn Energy, LLC", exchange:"NasdaqNM", type:"stock" }, -{ name: "LLY", description:"Eli Lilly and Company", exchange:"NYSE", type:"stock" }, -{ name: "LULU", description:"Lululemon Athletica Inc.", exchange:"NasdaqNM", type:"stock" }, -{ name: "LUV", description:"Southwest Airlines Co.", exchange:"NYSE", type:"stock" }, -{ name: "LYG", description:"Lloyds Banking Group plc", exchange:"NYSE", type:"stock" }, -{ name: "MCD", description:"McDonald's Corp.", exchange:"NYSE", type:"stock" }, -{ name: "MNST", description:"Monster Beverage Corporation", exchange:"NasdaqNM", type:"stock" }, -{ name: "MO", description:"Altria Group Inc.", exchange:"NYSE", type:"stock" }, -{ name: "MPEL", description:"Melco Crown Entertainment Limited", exchange:"NasdaqNM", type:"stock" }, -{ name: "MSFT", description:"Microsoft Corporation", exchange:"NasdaqNM", type:"stock" }, -{ name: "NLY", description:"Annaly Capital Management, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "NUS", description:"Nu Skin Enterprises Inc.", exchange:"NYSE", type:"stock" }, -{ name: "OLED", description:"Universal Display Corp.", exchange:"NasdaqNM", type:"stock" }, -{ name: "PNRA", description:"Panera Bread Company", exchange:"NasdaqNM", type:"stock" }, -{ name: "PRAN", description:"Prana Biotechnology Limited", exchange:"NCM", type:"stock" }, -{ name: "RAD", description:"Rite Aid Corporation", exchange:"NYSE", type:"stock" }, -{ name: "SAM", description:"Boston Beer Co. Inc.", exchange:"NYSE", type:"stock" }, -{ name: "SAN", description:"Banco Santander, S.A.", exchange:"NYSE", type:"stock" }, -{ name: "SCTY", description:"SolarCity Corporation", exchange:"NasdaqNM", type:"stock" }, -{ name: "SD", description:"SandRidge Energy, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "STZ", description:"Constellation Brands Inc.", exchange:"NYSE", type:"stock" }, -{ name: "SU", description:"Suncor Energy Inc.", exchange:"NYSE", type:"stock" }, -{ name: "T", description:"AT&T, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "TEG.AX", description:"Triangle Energy (Global) Limited", exchange:"ASX", type:"stock" }, -{ name: "TTW.V", description:"TIMES THREE WIRELESS INC", exchange:"CDNX", type:"stock" }, -{ name: "UA", description:"Under Armour, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "UCD.SG", description:"UC RESOURCES", exchange:"Stuttgart", type:"stock" }, -{ name: "USB", description:"U.S. Bancorp", exchange:"NYSE", type:"stock" }, -{ name: "VLG.MC", description:"", exchange:"NYSE", type:"stock" }, -{ name: "VZ", description:"Verizon Communications Inc.", exchange:"NYSE", type:"stock" }, -{ name: "WDC", description:"Western Digital Corporation", exchange:"NasdaqNM", type:"stock" }, -{ name: "WFC", description:"Wells Fargo & Company", exchange:"NYSE", type:"stock" }, -{ name: "WLT", description:"Walter Energy, Inc.", exchange:"NYSE", type:"stock" }, -{ name: "XOM", description:"Exxon Mobil Corporation", exchange:"NYSE", type:"stock" }, -]; +// This list should contain all the symbols available through your datafeed. +// The current version is extremely incomplete (as it's just a sample): Quandl has much more of them. + + +var symbols = [{"name":"A","description":"Agilent Technologies Inc.","exchange":"NYSE","type":"stock"}, +{"name":"AA","description":"Alcoa Inc.","exchange":"NYSE","type":"stock"}, +{"name":"AAL","description":"American Airlines Group Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AAPL","description":"Apple Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ABBV","description":"AbbVie Inc.","exchange":"NYSE","type":"stock"}, +{"name":"ABT","description":"Abbott Laboratories","exchange":"NYSE","type":"stock"}, +{"name":"ACHN","description":"Achillion Pharmaceuticals, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ACI","description":"Arch Coal Inc.","exchange":"NYSE","type":"stock"}, +{"name":"ACN","description":"Accenture plc","exchange":"NYSE","type":"stock"}, +{"name":"ADBE","description":"Adobe Systems Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ADSK","description":"Autodesk, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AEO","description":"American Eagle Outfitters, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"AGNC","description":"American Capital Agency Corp.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AIG","description":"American International Group, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"AKAM","description":"Akamai Technologies, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ALXN","description":"Alexion Pharmaceuticals, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AMAT","description":"Applied Materials, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AMD","description":"Advanced Micro Devices, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"AMGN","description":"Amgen Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AMZN","description":"Amazon.com Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ANF","description":"Abercrombie & Fitch Co.","exchange":"NYSE","type":"stock"}, +{"name":"ANR","description":"Alpha Natural Resources, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"APA","description":"Apache Corp.","exchange":"NYSE","type":"stock"}, +{"name":"APC","description":"Anadarko Petroleum Corporation","exchange":"NYSE","type":"stock"}, +{"name":"ARC","description":"ARC Document Solutions, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"ARIA","description":"Ariad Pharmaceuticals Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ARNA","description":"Arena Pharmaceuticals, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ARR","description":"ARMOUR Residential REIT, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"AUXL","description":"Auxilium Pharmaceuticals Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AVGO","description":"Avago Technologies Limited","exchange":"NasdaqNM","type":"stock"}, +{"name":"AVNR","description":"Avanir Pharmaceuticals, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AWAY","description":"HomeAway, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"AXP","description":"American Express Company","exchange":"NYSE","type":"stock"}, +{"name":"AZO","description":"AutoZone, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"BA","description":"The Boeing Company","exchange":"NYSE","type":"stock"}, +{"name":"BAC","description":"Bank of America Corporation","exchange":"NYSE","type":"stock"}, +{"name":"BAX","description":"Baxter International Inc.","exchange":"NYSE","type":"stock"}, +{"name":"BBBY","description":"Bed Bath & Beyond Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"BBT","description":"BB&T Corporation","exchange":"NYSE","type":"stock"}, +{"name":"BBY","description":"Best Buy Co., Inc.","exchange":"NYSE","type":"stock"}, +{"name":"BIDU","description":"Baidu, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"BIIB","description":"Biogen Idec Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"BK","description":"The Bank of New York Mellon Corporation","exchange":"NYSE","type":"stock"}, +{"name":"BLK","description":"BlackRock, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"BMY","description":"Bristol-Myers Squibb Company","exchange":"NYSE","type":"stock"}, +{"name":"BP","description":"BP plc","exchange":"NYSE","type":"stock"}, +{"name":"BRCD","description":"Brocade Communications Systems, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"BRCM","description":"Broadcom Corp.","exchange":"NasdaqNM","type":"stock"}, +{"name":"BTU","description":"Peabody Energy Corp.","exchange":"NYSE","type":"stock"}, +{"name":"C","description":"Citigroup Inc.","exchange":"NYSE","type":"stock"}, +{"name":"CHK","description":"Chesapeake Energy Corporation","exchange":"NYSE","type":"stock"}, +{"name":"CNP","description":"CenterPoint Energy, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"CSCO","description":"Cisco Systems, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"D","description":"Dominion Resources, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"DAL","description":"Delta Air Lines Inc.","exchange":"NYSE","type":"stock"}, +{"name":"DBD","description":"Diebold, Incorporated","exchange":"NYSE","type":"stock"}, +{"name":"DD","description":"E. I. du Pont de Nemours and Company","exchange":"NYSE","type":"stock"}, +{"name":"DDD","description":"3D Systems Corp.","exchange":"NYSE","type":"stock"}, +{"name":"DE","description":"Deere & Company","exchange":"NYSE","type":"stock"}, +{"name":"DECK","description":"Deckers Outdoor Corp.","exchange":"NYSE","type":"stock"}, +{"name":"DEI","description":"Douglas Emmett Inc","exchange":"NYSE","type":"stock"}, +{"name":"DHI","description":"DR Horton Inc.","exchange":"NYSE","type":"stock"}, +{"name":"DIS","description":"The Walt Disney Company","exchange":"NYSE","type":"stock"}, +{"name":"DLTR","description":"Dollar Tree, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"DNDN","description":"Dendreon Corp.","exchange":"NasdaqNM","type":"stock"}, +{"name":"DO","description":"Diamond Offshore Drilling, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"DOV","description":"Dover Corporation","exchange":"NYSE","type":"stock"}, +{"name":"DOW","description":"The Dow Chemical Company","exchange":"NYSE","type":"stock"}, +{"name":"DRI","description":"Darden Restaurants, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"DV","description":"DeVry Education Group Inc.","exchange":"NYSE","type":"stock"}, +{"name":"DVN","description":"Devon Energy Corporation","exchange":"NYSE","type":"stock"}, +{"name":"EA","description":"Electronic Arts Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"EBAY","description":"eBay Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"EBIX","description":"Ebix Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ECYT","description":"Endocyte, Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"ED","description":"Consolidated Edison, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"EMC","description":"EMC Corporation","exchange":"NYSE","type":"stock"}, +{"name":"ENT","description":"Global Eagle Entertainment Inc.","exchange":"NCM","type":"stock"}, +{"name":"ESI","description":"ITT Educational Services Inc.","exchange":"NYSE","type":"stock"}, +{"name":"ESRX","description":"Express Scripts Holding Company","exchange":"NasdaqNM","type":"stock"}, +{"name":"ETFC","description":"E*TRADE Financial Corporation","exchange":"NasdaqNM","type":"stock"}, +{"name":"EXC","description":"Exelon Corporation","exchange":"NYSE","type":"stock"}, +{"name":"EXPE","description":"Expedia Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"F","description":"Ford Motor Co.","exchange":"NYSE","type":"stock"}, +{"name":"FCEL","description":"FuelCell Energy Inc.","exchange":"NGM","type":"stock"}, +{"name":"GALE","description":"Galena Biopharma, Inc.","exchange":"NCM","type":"stock"}, +{"name":"GD","description":"General Dynamics Corp.","exchange":"NYSE","type":"stock"}, +{"name":"GE","description":"General Electric Company","exchange":"NYSE","type":"stock"}, +{"name":"GTAT","description":"GT Advanced Technologies Inc.","exchange":"NasdaqNM","type":"stock"}, +{"name":"HD","description":"The Home Depot, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"IBM","description":"International Business Machines Corporation","exchange":"NYSE","type":"stock"}, +{"name":"INTC","description":"Intel Corporation","exchange":"NasdaqNM","type":"stock"}, +{"name":"JPM","description":"JPMorgan Chase & Co.","exchange":"NYSE","type":"stock"}, +{"name":"KERX","description":"Keryx Biopharmaceuticals Inc.","exchange":"NCM","type":"stock"}, +{"name":"KO","description":"The Coca-Cola Company","exchange":"NYSE","type":"stock"}, +{"name":"LLY","description":"Eli Lilly and Company","exchange":"NYSE","type":"stock"}, +{"name":"LUV","description":"Southwest Airlines Co.","exchange":"NYSE","type":"stock"}, +{"name":"MCD","description":"McDonald's Corp.","exchange":"NYSE","type":"stock"}, +{"name":"MNST","description":"Monster Beverage Corporation","exchange":"NasdaqNM","type":"stock"}, +{"name":"MO","description":"Altria Group Inc.","exchange":"NYSE","type":"stock"}, +{"name":"MSFT","description":"Microsoft Corporation","exchange":"NasdaqNM","type":"stock"}, +{"name":"NLY","description":"Annaly Capital Management, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"NUS","description":"Nu Skin Enterprises Inc.","exchange":"NYSE","type":"stock"}, +{"name":"OLED","description":"Universal Display Corp.","exchange":"NasdaqNM","type":"stock"}, +{"name":"PNRA","description":"Panera Bread Company","exchange":"NasdaqNM","type":"stock"}, +{"name":"RAD","description":"Rite Aid Corporation","exchange":"NYSE","type":"stock"}, +{"name":"SAM","description":"Boston Beer Co. Inc.","exchange":"NYSE","type":"stock"}, +{"name":"SCTY","description":"SolarCity Corporation","exchange":"NasdaqNM","type":"stock"}, +{"name":"SD","description":"SandRidge Energy, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"STZ","description":"Constellation Brands Inc.","exchange":"NYSE","type":"stock"}, +{"name":"T","description":"AT&T, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"UA","description":"Under Armour, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"USB","description":"U.S. Bancorp","exchange":"NYSE","type":"stock"}, +{"name":"VZ","description":"Verizon Communications Inc.","exchange":"NYSE","type":"stock"}, +{"name":"WDC","description":"Western Digital Corporation","exchange":"NasdaqNM","type":"stock"}, +{"name":"WFC","description":"Wells Fargo & Company","exchange":"NYSE","type":"stock"}, +{"name":"WLT","description":"Walter Energy, Inc.","exchange":"NYSE","type":"stock"}, +{"name":"XOM","description":"Exxon Mobil Corporation","exchange":"NYSE","type":"stock"}]; function searchResultFromDatabaseItem(item) { diff --git a/yahoo.js b/yahoo.js index 0ba19b5..315f0e5 100644 --- a/yahoo.js +++ b/yahoo.js @@ -1,19 +1,22 @@ /* This file is a node.js module. - This is a sample implementation of UDF-compatible datafeed wrapper for yahoo.finance. - Some algorithms may be icorrect because it's rather an UDF implementation sample + This is a sample implementation of UDF-compatible datafeed wrapper for Quandl (historical data) and yahoo.finance (quotes). + Some algorithms may be incorrect because it's rather an UDF implementation sample then a proper datafeed implementation. */ +/* global require */ +/* global console */ +/* global process */ + +"use strict"; + var http = require("http"), https = require("https"), url = require("url"), symbolsDatabase = require("./symbols_database"); -var datafeedHost = "chartapi.finance.yahoo.com"; -var lastHistoryErrorTime = null; -var errorSwitchingTime = 60 * 60 * 1000; // switch to Quandl for 1 hour var quandlCache = {}; var quandlCacheCleanupTime = 3 * 60 * 60 * 100; // 3 hours @@ -27,27 +30,27 @@ function createDefaultHeader() { var defaultResponseHeader = createDefaultHeader(); -function httpGet(datafeedHost, path, callback, failedCallback) +function httpGet(datafeedHost, path, callback) { var options = { host: datafeedHost, path: path }; - onDataCallback = function(response) { + function onDataCallback(response) { var result = ''; response.on('data', function (chunk) { - result += chunk + result += chunk; }); response.on('end', function () { if (response.statusCode !== 200) { - failedCallback ? failedCallback(response.statusCode) : callback(''); + callback(''); return; } - callback(result) + callback(result); }); } @@ -63,49 +66,12 @@ function httpGet(datafeedHost, path, callback, failedCallback) req.on('error', function(e) { console.log('Problem with request: ' + e.message); - failedCallback ? failedCallback(e) : callback(''); + callback(''); }); req.end(); } - -function convertYahooHistoryToUDFFormat(data) { - - // input: string "yyyy-mm-dd" (UTC) - // output: milliseconds from 01.01.1970 00:00:00.000 UTC - function parseDate(input) { - var parts = input.split('-'); - return Date.UTC(parts[0], parts[1]-1, parts[2]); - } - - var result = { - t: [], c: [], o: [], h: [], l: [], v: [], - s: "ok" - }; - - var lines = data.split('\n'); - - for (var i = lines.length - 2; i > 0; --i) { - var items = lines[i].split(","); - - var time = parseDate(items[0]) / 1000; - - result.t.push(time); - result.o.push(parseFloat(items[1])); - result.h.push(parseFloat(items[2])); - result.l.push(parseFloat(items[3])); - result.c.push(parseFloat(items[4])); - result.v.push(parseFloat(items[5])); - } - - if (result.t.length === 0) { - result.s = "no_data"; - } - - return result; -} - function convertQuandlHistoryToUDFFormat(data) { function parseDate(input) { var parts = input.split('-'); @@ -129,11 +95,9 @@ function convertQuandlHistoryToUDFFormat(data) { try { var json = JSON.parse(data); var datatable = json.datatable; - var data = datatable.data; - var columns = datatable.columns; - var idx = columnIndices(columns); - - data.forEach(function(row) { + var idx = columnIndices(datatable.columns); + + datatable.data.forEach(function(row) { result.t.push(parseDate(row[idx.date]) / 1000); result.o.push(row[idx.open]); result.h.push(row[idx.high]); @@ -167,7 +131,7 @@ function convertYahooQuotesToUDFFormat(tickersMap, data) { } result.d.push({ - s: "ok", + s: "ok", n: ticker, v: { ch: quote.ChangeRealtime || quote.Change, @@ -188,7 +152,7 @@ function convertYahooQuotesToUDFFormat(tickersMap, data) { prev_close_price: quote.PreviousClose, volume: quote.Volume, } - }); + }); }); return result; } @@ -215,7 +179,7 @@ function proxyRequest(controller, options, response) { }).end(); } -RequestProcessor = function(action, query, response) { +function RequestProcessor(action, query, response) { this.sendError = function(error, response) { response.writeHead(200, defaultResponseHeader); @@ -223,7 +187,7 @@ RequestProcessor = function(action, query, response) { response.end(); console.log(error); - } + }; this.sendConfig = function(response) { @@ -235,13 +199,11 @@ RequestProcessor = function(action, query, response) { supports_timescale_marks: true, supports_time: true, exchanges: [ - {value: "", name: "All Exchanges", desc: ""}, - {value: "XETRA", name: "XETRA", desc: "XETRA"}, - {value: "NSE", name: "NSE", desc: "NSE"}, + {value: "", name: "All Exchanges", desc: ""}, {value: "NasdaqNM", name: "NasdaqNM", desc: "NasdaqNM"}, {value: "NYSE", name: "NYSE", desc: "NYSE"}, - {value: "CDNX", name: "CDNX", desc: "CDNX"}, - {value: "Stuttgart", name: "Stuttgart", desc: "Stuttgart"}, + {value: "NCM", name: "NCM", desc: "NCM"}, + {value: "NGM", name: "NGM", desc: "NGM"}, ], symbolsTypes: [ {name: "All types", value: ""}, @@ -254,7 +216,7 @@ RequestProcessor = function(action, query, response) { response.writeHead(200, defaultResponseHeader); response.write(JSON.stringify(config)); response.end(); - } + }; this.sendMarks = function(response) { @@ -275,7 +237,7 @@ RequestProcessor = function(action, query, response) { response.writeHead(200, defaultResponseHeader); response.write(JSON.stringify(marks)); response.end(); - } + }; this.sendTime = function(response) { var now = new Date(); @@ -313,26 +275,12 @@ RequestProcessor = function(action, query, response) { response.writeHead(200, defaultResponseHeader); response.write(JSON.stringify(result)); response.end(); - } - - - this._pendingRequestType = ""; - this._lastYahooResponse = null; - - this.finance_charts_json_callback = function(data) { - if (_pendingRequestType == "data") { - _lastYahooResponse = data.series; - } - else if (_pendingRequestType == "meta") { - _lastYahooResponse = data.meta; - } - } - + }; this.sendSymbolInfo = function(symbolName, response) { var symbolInfo = symbolsDatabase.symbolInfo(symbolName); - if (symbolInfo == null) { + if (!symbolInfo) { throw "unknown_symbol " + symbolName; } @@ -346,69 +294,24 @@ RequestProcessor = function(action, query, response) { "pointvalue": 1, "session": "0930-1630", "has_intraday": false, - "has_no_volume": symbolInfo.type != "stock", + "has_no_volume": symbolInfo.type !== "stock", "description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name, "type": symbolInfo.type, "supported_resolutions" : ["D","2D","3D","W","3W","M","6M"], "pricescale": 100, - "ticker": symbolInfo.name.toUpperCase(), - }; - - if (lastHistoryErrorTime && Date.now() - lastHistoryErrorTime < errorSwitchingTime) { - // return default response if we have problems with Yahoo - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(info)); - response.end(); - return; - } - - var address = "/instrument/1.0/" + encodeURIComponent(symbolInfo.name) + "/chartdata;type=quote;/json"; - var that = this; - - console.log(datafeedHost + address); - - httpGet(datafeedHost, address, function(result) { - _pendingRequestType = "meta"; - - try { - with (that) { - eval(result); - } - } - catch (error) { - that.sendError("invalid symbol", response); - return; - } - - try { - var lastPrice = _lastYahooResponse["previous_close"] + ""; - - // BEWARE: this `pricescale` parameter computation algorithm is wrong and works - // for symbols with 10-based minimal movement value only - var pricescale = lastPrice.indexOf('.') > 0 - ? Math.pow(10, lastPrice.split('.')[1].length) - : 10; - - Object.assign(info, { - "pricescale": pricescale, - "ticker": _lastYahooResponse["ticker"].toUpperCase(), - }); - } catch(error) { - console.error(error); - } - + "ticker": symbolInfo.name.toUpperCase() + }; - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(info)); - response.end(); - }); - } + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(info)); + response.end(); + }; - function requestHistoryFromQuandl(symbol, startDateTimestamp, endDateTimestamp, response) { + this.sendSymbolHistory = function(symbol, startDateTimestamp, endDateTimestamp, resolution, response) { function dateToYMD(date) { var obj = new Date(date * 1000); var year = obj.getFullYear(); - var month = obj.getMonth(); + var month = obj.getMonth() + 1; var day = obj.getDate(); return year + "-" + month + "-" + day; } @@ -450,71 +353,8 @@ RequestProcessor = function(action, query, response) { quandlCache[key] = content; sendResult(content); }); - }; - this.sendSymbolHistory = function(symbol, startDateTimestamp, endDateTimestamp, resolution, response) { - if (lastHistoryErrorTime && Date.now() - lastHistoryErrorTime < errorSwitchingTime) { - requestHistoryFromQuandl(symbol, startDateTimestamp, endDateTimestamp, response); - return; - } - - var symbolInfo = symbolsDatabase.symbolInfo(symbol); - - if (symbolInfo == null) { - throw "unknown_symbol"; - } - - var requestLeftDate = new Date(startDateTimestamp * 1000); - console.log(requestLeftDate); - - var year = requestLeftDate.getFullYear(); - var month = requestLeftDate.getMonth(); - var day = requestLeftDate.getDate(); - - var endtext = ''; - - if (endDateTimestamp) { - var requestRightDate = new Date(endDateTimestamp * 1000); - var endyear = requestRightDate.getFullYear(); - var endmonth = requestRightDate.getMonth(); - var endday = requestRightDate.getDate(); - - endtext = '&d=' + endmonth + - '&e=' + endday + - '&f=' + endyear; - } - - if (resolution != "d" && resolution != "w" && resolution != "m") { - throw "Unsupported resolution: " + resolution; - } - - var address = "ichart.finance.yahoo.com/table.csv?s=" + symbolInfo.name + - "&a=" + month + - "&b=" + day + - "&c=" + year + endtext + - "&g=" + resolution + - "&ignore=.csv"; - - console.log("Requesting " + address); - - var that = this; - - httpGet(datafeedHost, address, function(result) { - var content = JSON.stringify(convertYahooHistoryToUDFFormat(result)); - var header = createDefaultHeader(); - header["Content-Length"] = content.length; - response.writeHead(200, header); - response.write(content, null, function() { - response.end(); - }); - }, function(error) { - // try another feed - requestHistoryFromQuandl(symbol, startDateTimestamp, endDateTimestamp, response); - lastHistoryErrorTime = Date.now(); - }); - } - this.sendQuotes = function(tickersString, response) { var tickersMap = {}; // maps YQL symbol to ticker @@ -529,9 +369,9 @@ RequestProcessor = function(action, query, response) { var options = { host: "query.yahooapis.com", - path: "/v1/public/yql?q=" + encodeURIComponent(yql) - + "&format=json" - + "&env=store://datatables.org/alltableswithkeys", + path: "/v1/public/yql?q=" + encodeURIComponent(yql) + + "&format=json" + + "&env=store://datatables.org/alltableswithkeys" }; // for debug purposes // console.log(options.host + options.path); @@ -556,61 +396,61 @@ RequestProcessor = function(action, query, response) { response.end(); }); }).end(); - } + }; this.sendNews = function(symbol, response) { var options = { host: "feeds.finance.yahoo.com", - path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US", + path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US" }; proxyRequest(https, options, response); - } + }; this.sendFuturesmag = function(response) { var options = { host: "www.futuresmag.com", - path: "/rss/all", + path: "/rss/all" }; proxyRequest(http, options, response); - } + }; try { - if (action == "/config") { + if (action === "/config") { this.sendConfig(response); } - else if (action == "/symbols" && !!query["symbol"]) { + else if (action === "/symbols" && !!query["symbol"]) { this.sendSymbolInfo(query["symbol"], response); } - else if (action == "/search") { + else if (action === "/search") { this.sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); } - else if (action == "/history") { + else if (action === "/history") { this.sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response); } - else if (action == "/quotes") { + else if (action === "/quotes") { this.sendQuotes(query["symbols"], response); } - else if (action == "/marks") { + else if (action === "/marks") { this.sendMarks(response); } - else if (action == "/time") { + else if (action === "/time") { this.sendTime(response); } - else if (action == "/timescale_marks") { + else if (action === "/timescale_marks") { this.sendTimescaleMarks(response); } - else if (action == "/news") { + else if (action === "/news") { this.sendNews(query["symbol"], response); } - else if (action == "/futuresmag") { + else if (action === "/futuresmag") { this.sendFuturesmag(response); } } catch (error) { - this.sendError(error, response) + this.sendError(error, response); } } @@ -644,7 +484,7 @@ getFreePort(function(port) { http.createServer(function(request, response) { var uri = url.parse(request.url, true); var action = uri.pathname; - new RequestProcessor(action, uri.query, response); + return new RequestProcessor(action, uri.query, response); }).listen(port); From 66f4ad59453acafd51e6a8f1636c6615fe5d169b Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 20 Jul 2017 16:17:57 +0300 Subject: [PATCH 22/59] prepare for emulation branch --- request-processor.js | 550 +++++++++++++++++++++++++++++++++++++++++++ symbols_database.js | 15 +- yahoo.js | 444 +--------------------------------- 3 files changed, 565 insertions(+), 444 deletions(-) create mode 100644 request-processor.js diff --git a/request-processor.js b/request-processor.js new file mode 100644 index 0000000..04154bc --- /dev/null +++ b/request-processor.js @@ -0,0 +1,550 @@ +/* + This file is a node.js module. + + This is a sample implementation of UDF-compatible datafeed wrapper for Quandl (historical data) and yahoo.finance (quotes). + Some algorithms may be incorrect because it's rather an UDF implementation sample + then a proper datafeed implementation. +*/ + +/* global require */ +/* global console */ +/* global exports */ +/* global process */ + +"use strict"; + +var https = require("https"); +var http = require("http"); + +var quandlCache = {}; + +var quandlCacheCleanupTime = 3 * 60 * 60 * 100; // 3 hours + +// this cache is intended to reduce number of requests to Quandl +setInterval(function () { + quandlCache = {}; +}, quandlCacheCleanupTime); + +function createDefaultHeader() { + return { + "Content-Type": "text/plain", + 'Access-Control-Allow-Origin': '*' + }; +} + +var defaultResponseHeader = createDefaultHeader(); + +function httpGet(datafeedHost, path, callback) { + var options = { + host: datafeedHost, + path: path + }; + + function onDataCallback(response) { + var result = ''; + + response.on('data', function (chunk) { + result += chunk; + }); + + response.on('end', function () { + if (response.statusCode !== 200) { + callback(''); + return; + } + + callback(result); + }); + } + + var req = https.request(options, onDataCallback); + + req.on('socket', function (socket) { + socket.setTimeout(5000); + socket.on('timeout', function () { + console.log('timeout'); + req.abort(); + }); + }); + + req.on('error', function (e) { + console.log('Problem with request: ' + e.message); + callback(''); + }); + + req.end(); +} + +function convertQuandlHistoryToUDFFormat(data) { + function parseDate(input) { + var parts = input.split('-'); + return Date.UTC(parts[0], parts[1] - 1, parts[2]); + } + + function columnIndices(columns) { + var indices = {}; + for (var i = 0; i < columns.length; i++) { + indices[columns[i].name] = i; + } + + return indices; + } + + var result = { + t: [], + c: [], + o: [], + h: [], + l: [], + v: [], + s: "ok" + }; + + try { + var json = JSON.parse(data); + var datatable = json.datatable; + var idx = columnIndices(datatable.columns); + + datatable.data.forEach(function (row) { + result.t.push(parseDate(row[idx.date]) / 1000); + result.o.push(row[idx.open]); + result.h.push(row[idx.high]); + result.l.push(row[idx.low]); + result.c.push(row[idx.close]); + result.v.push(row[idx.volume]); + }); + + } catch (error) { + console.error(error); + } + + return result; +} + +function convertYahooQuotesToUDFFormat(tickersMap, data) { + if (!data.query || !data.query.results) { + var errmsg = "ERROR: empty quotes response: " + JSON.stringify(data); + console.log(errmsg); + return { + s: "error", + errmsg: errmsg + }; + } + + var result = { + s: "ok", + d: [] + }; + [].concat(data.query.results.quote).forEach(function (quote) { + var ticker = tickersMap[quote.symbol]; + + // this field is an error token + if (quote["ErrorIndicationreturnedforsymbolchangedinvalid"] || !quote.StockExchange) { + result.d.push({ + s: "error", + n: ticker, + v: {} + }); + return; + } + + result.d.push({ + s: "ok", + n: ticker, + v: { + ch: quote.ChangeRealtime || quote.Change, + chp: (quote.PercentChange || quote.ChangeinPercent) && (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1"), + + short_name: quote.Symbol, + exchange: quote.StockExchange, + original_name: quote.StockExchange + ":" + quote.Symbol, + description: quote.Name, + + lp: quote.LastTradePriceOnly, + ask: quote.AskRealtime, + bid: quote.BidRealtime, + + open_price: quote.Open, + high_price: quote.DaysHigh, + low_price: quote.DaysLow, + prev_close_price: quote.PreviousClose, + volume: quote.Volume, + } + }); + }); + return result; +} + +function proxyRequest(controller, options, response) { + controller.request(options, function (res) { + var result = ''; + + res.on('data', function (chunk) { + result += chunk; + }); + + res.on('end', function () { + if (res.statusCode !== 200) { + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify({ + s: 'error', + errmsg: 'Failed to get news' + })); + response.end(); + return; + } + response.writeHead(200, defaultResponseHeader); + response.write(result); + response.end(); + }); + }).end(); +} + +function RequestProcessor(symbolsDatabase) { + this._symbolsDatabase = symbolsDatabase; +} + +RequestProcessor.prototype._sendError = function (error, response) { + response.writeHead(200, defaultResponseHeader); + response.write("{\"s\":\"error\",\"errmsg\":\"" + error + "\"}"); + response.end(); + + console.log(error); +}; + + +RequestProcessor.prototype._sendConfig = function (response) { + + var config = { + supports_search: true, + supports_group_request: false, + supports_marks: true, + supports_timescale_marks: true, + supports_time: true, + exchanges: [ + { + value: "", + name: "All Exchanges", + desc: "" + }, + { + value: "NasdaqNM", + name: "NasdaqNM", + desc: "NasdaqNM" + }, + { + value: "NYSE", + name: "NYSE", + desc: "NYSE" + }, + { + value: "NCM", + name: "NCM", + desc: "NCM" + }, + { + value: "NGM", + name: "NGM", + desc: "NGM" + }, + ], + symbolsTypes: [ + { + name: "All types", + value: "" + }, + { + name: "Stock", + value: "stock" + }, + { + name: "Index", + value: "index" + } + ], + supportedResolutions: ["D", "2D", "3D", "W", "3W", "M", '6M'] + }; + + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(config)); + response.end(); +}; + + +RequestProcessor.prototype._sendMarks = function (response) { + var now = new Date(); + now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; + var day = 60 * 60 * 24; + + var marks = { + id: [0, 1, 2, 3, 4, 5], + time: [now, now - day * 4, now - day * 7, now - day * 7, now - day * 15, now - day * 30], + color: ["red", "blue", "green", "red", "blue", "green"], + text: ["Today", "4 days back", "7 days back + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", "7 days back once again", "15 days back", "30 days back"], + label: ["A", "B", "CORE", "D", "EURO", "F"], + labelFontColor: ["white", "white", "red", "#FFFFFF", "white", "#000"], + minSize: [14, 28, 7, 40, 7, 14] + }; + + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(marks)); + response.end(); +}; + +RequestProcessor.prototype._sendTime = function (response) { + var now = new Date(); + response.writeHead(200, defaultResponseHeader); + response.write(Math.floor(now / 1000) + ''); + response.end(); +}; + +RequestProcessor.prototype._sendTimescaleMarks = function (response) { + var now = new Date(); + now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; + var day = 60 * 60 * 24; + + var marks = [ + { + id: "tsm1", + time: now - day * 0, + color: "red", + label: "A", + tooltip: "" + }, + { + id: "tsm2", + time: now - day * 4, + color: "blue", + label: "D", + tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()] + }, + { + id: "tsm3", + time: now - day * 7, + color: "green", + label: "D", + tooltip: ["Dividends: $3.46", "Date: " + new Date((now - day * 7) * 1000).toDateString()] + }, + { + id: "tsm4", + time: now - day * 15, + color: "#999999", + label: "E", + tooltip: ["Earnings: $3.44", "Estimate: $3.60"] + }, + { + id: "tsm7", + time: now - day * 30, + color: "red", + label: "E", + tooltip: ["Earnings: $5.40", "Estimate: $5.00"] + }, + ]; + + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(marks)); + response.end(); +}; + + +RequestProcessor.prototype._sendSymbolSearchResults = function (query, type, exchange, maxRecords, response) { + if (!maxRecords) { + throw "wrong_query"; + } + + var result = this._symbolsDatabase.search(query, type, exchange, maxRecords); + + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(result)); + response.end(); +}; + +RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { + var symbolInfo = this._symbolsDatabase.symbolInfo(symbolName); + + if (!symbolInfo) { + throw "unknown_symbol " + symbolName; + } + + return { + "name": symbolInfo.name, + "exchange-traded": symbolInfo.exchange, + "exchange-listed": symbolInfo.exchange, + "timezone": "America/New_York", + "minmov": 1, + "minmov2": 0, + "pointvalue": 1, + "session": "0930-1630", + "has_intraday": false, + "has_no_volume": symbolInfo.type !== "stock", + "description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name, + "type": symbolInfo.type, + "supported_resolutions": ["D", "2D", "3D", "W", "3W", "M", "6M"], + "pricescale": 100, + "ticker": symbolInfo.name.toUpperCase() + }; +}; + +RequestProcessor.prototype._sendSymbolInfo = function (symbolName, response) { + var info = this._prepareSymbolInfo(symbolName); + + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(info)); + response.end(); +}; + +RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimestamp, endDateTimestamp, resolution, response) { + function dateToYMD(date) { + var obj = new Date(date * 1000); + var year = obj.getFullYear(); + var month = obj.getMonth() + 1; + var day = obj.getDate(); + return year + "-" + month + "-" + day; + } + + function sendResult(content) { + var header = createDefaultHeader(); + header["Content-Length"] = content.length; + response.writeHead(200, header); + response.write(content, null, function () { + response.end(); + }); + } + + var from = '1970-01-01'; // dateToYMD(startDateTimestamp); always return all data to reduce number of requests to quandl + var to = dateToYMD(endDateTimestamp); + + var key = symbol + "|" + from + "|" + to; + + if (quandlCache[key]) { + console.log("Return QUANDL result from cache: " + key); + sendResult(quandlCache[key]); + return; + } + + var address = "/api/v3/datatables/WIKI/PRICES.json" + + "?api_key=" + process.env.QUANDL_API_KEY + // you should create a free account on quandl.com to get this key + "&ticker=" + symbol + + "&date.gte=" + from + + "&date.lte=" + to; + + console.log("Sending request to quandl for symbol " + symbol + ". url=" + address); + + httpGet("www.quandl.com", address, function (result) { + if (response.finished) { + // we can be here if error happened on socket disconnect + return; + } + var content = JSON.stringify(convertQuandlHistoryToUDFFormat(result)); + quandlCache[key] = content; + sendResult(content); + }); +}; + +RequestProcessor.prototype._sendQuotes = function (tickersString, response) { + var tickersMap = {}; // maps YQL symbol to ticker + + var tickers = tickersString.split(","); + [].concat(tickers).forEach(function (ticker) { + var yqlSymbol = ticker.replace(/.*:(.*)/, "$1"); + tickersMap[yqlSymbol] = ticker; + }); + + var yql = "select * from yahoo.finance.quotes where symbol in ('" + Object.keys(tickersMap).join("','") + "')"; + console.log("Quotes query: " + yql); + + var options = { + host: "query.yahooapis.com", + path: "/v1/public/yql?q=" + encodeURIComponent(yql) + + "&format=json" + + "&env=store://datatables.org/alltableswithkeys" + }; + // for debug purposes + // console.log(options.host + options.path); + + http.request(options, function (res) { + var result = ''; + + res.on('data', function (chunk) { + result += chunk; + }); + + res.on('end', function () { + if (res.statusCode !== 200) { + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify({ + s: 'error', + errmsg: 'Yahoo fails' + })); + response.end(); + return; + } + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(convertYahooQuotesToUDFFormat( + tickersMap, JSON.parse(result)))); + response.end(); + }); + }).end(); +}; + +RequestProcessor.prototype._sendNews = function (symbol, response) { + var options = { + host: "feeds.finance.yahoo.com", + path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US" + }; + + proxyRequest(https, options, response); +}; + +RequestProcessor.prototype._sendFuturesmag = function (response) { + var options = { + host: "www.futuresmag.com", + path: "/rss/all" + }; + + proxyRequest(http, options, response); +}; + +RequestProcessor.prototype._defaultResponseHeader = function() { + return defaultResponseHeader; +}; + +RequestProcessor.prototype.processRequest = function (action, query, response) { + try { + if (action === "/config") { + this._sendConfig(response); + } + else if (action === "/symbols" && !!query["symbol"]) { + this._sendSymbolInfo(query["symbol"], response); + } + else if (action === "/search") { + this._sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); + } + else if (action === "/history") { + this._sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response); + } + else if (action === "/quotes") { + this._sendQuotes(query["symbols"], response); + } + else if (action === "/marks") { + this._sendMarks(response); + } + else if (action === "/time") { + this._sendTime(response); + } + else if (action === "/timescale_marks") { + this._sendTimescaleMarks(response); + } + else if (action === "/news") { + this._sendNews(query["symbol"], response); + } + else if (action === "/futuresmag") { + this._sendFuturesmag(response); + } + } + catch (error) { + this._sendError(error, response); + } +}; + +exports.RequestProcessor = RequestProcessor; \ No newline at end of file diff --git a/symbols_database.js b/symbols_database.js index 507112c..fad6bd1 100644 --- a/symbols_database.js +++ b/symbols_database.js @@ -4,6 +4,9 @@ // This list should contain all the symbols available through your datafeed. // The current version is extremely incomplete (as it's just a sample): Quandl has much more of them. +"use strict"; + +/* global exports */ var symbols = [{"name":"A","description":"Agilent Technologies Inc.","exchange":"NYSE","type":"stock"}, {"name":"AA","description":"Alcoa Inc.","exchange":"NYSE","type":"stock"}, @@ -139,7 +142,7 @@ function searchResultFromDatabaseItem(item) { exports.search = function (searchString, type, exchange, maxRecords) { var MAX_SEARCH_RESULTS = !!maxRecords ? maxRecords : 50; var results = []; // array of WeightedItem { item, weight } - var queryIsEmpty = !searchString || searchString.length == 0; + var queryIsEmpty = !searchString || searchString.length === 0; var searchStringUpperCase = searchString.toUpperCase(); for (var i = 0; i < symbols.length; ++i) { @@ -174,9 +177,13 @@ exports.search = function (searchString, type, exchange, maxRecords) { .sort(function (weightedItem1, weightedItem2) { return weightedItem1.weight - weightedItem2.weight; }) .map(function (weightedItem) { return searchResultFromDatabaseItem(weightedItem.item); }) .slice(0, Math.min(results.length, MAX_SEARCH_RESULTS)); -} +}; +exports.addSymbols = function(newSymbols) { + symbols = symbols.concat(newSymbols); +}; + exports.symbolInfo = function (symbolName) { var data = symbolName.split(':'); @@ -186,10 +193,10 @@ exports.symbolInfo = function (symbolName) { for (var i = 0; i < symbols.length; ++i) { var item = symbols[i]; - if (item.name.toUpperCase() == symbol && (exchange.length == 0 || exchange == item.exchange.toUpperCase())) { + if (item.name.toUpperCase() == symbol && (exchange.length === 0 || exchange == item.exchange.toUpperCase())) { return item; } } return null; -} +}; diff --git a/yahoo.js b/yahoo.js index 315f0e5..8ffd2b7 100644 --- a/yahoo.js +++ b/yahoo.js @@ -13,447 +13,11 @@ "use strict"; var http = require("http"), - https = require("https"), url = require("url"), - symbolsDatabase = require("./symbols_database"); - -var quandlCache = {}; - -var quandlCacheCleanupTime = 3 * 60 * 60 * 100; // 3 hours -setInterval(function() { - quandlCache = {}; -}, quandlCacheCleanupTime); - -function createDefaultHeader() { - return {"Content-Type": "text/plain", 'Access-Control-Allow-Origin': '*'}; -} - -var defaultResponseHeader = createDefaultHeader(); - -function httpGet(datafeedHost, path, callback) -{ - var options = { - host: datafeedHost, - path: path - }; - - function onDataCallback(response) { - var result = ''; - - response.on('data', function (chunk) { - result += chunk; - }); - - response.on('end', function () { - if (response.statusCode !== 200) { - callback(''); - return; - } - - callback(result); - }); - } - - var req = https.request(options, onDataCallback); - - req.on('socket', function (socket) { - socket.setTimeout(5000); - socket.on('timeout', function() { - console.log('timeout'); - req.abort(); - }); - }); - - req.on('error', function(e) { - console.log('Problem with request: ' + e.message); - callback(''); - }); - - req.end(); -} - -function convertQuandlHistoryToUDFFormat(data) { - function parseDate(input) { - var parts = input.split('-'); - return Date.UTC(parts[0], parts[1]-1, parts[2]); - } - - function columnIndices(columns) { - var indices = {}; - for (var i = 0; i < columns.length; i++) { - indices[columns[i].name] = i; - } - - return indices; - } - - var result = { - t: [], c: [], o: [], h: [], l: [], v: [], - s: "ok" - }; - - try { - var json = JSON.parse(data); - var datatable = json.datatable; - var idx = columnIndices(datatable.columns); - - datatable.data.forEach(function(row) { - result.t.push(parseDate(row[idx.date]) / 1000); - result.o.push(row[idx.open]); - result.h.push(row[idx.high]); - result.l.push(row[idx.low]); - result.c.push(row[idx.close]); - result.v.push(row[idx.volume]); - }); - - } catch(error) { - console.error(error); - } - - return result; -} - -function convertYahooQuotesToUDFFormat(tickersMap, data) { - if (!data.query || !data.query.results) { - var errmsg = "ERROR: empty quotes response: " + JSON.stringify(data); - console.log(errmsg); - return { s: "error", errmsg: errmsg }; - } - - var result = { s: "ok", d: [] }; - [].concat(data.query.results.quote).forEach(function(quote) { - var ticker = tickersMap[quote.symbol]; - - // this field is an error token - if (quote["ErrorIndicationreturnedforsymbolchangedinvalid"] || !quote.StockExchange) { - result.d.push({ s: "error", n: ticker, v: {} }); - return; - } - - result.d.push({ - s: "ok", - n: ticker, - v: { - ch: quote.ChangeRealtime || quote.Change, - chp: (quote.PercentChange || quote.ChangeinPercent) && (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1"), - - short_name: quote.Symbol, - exchange: quote.StockExchange, - original_name: quote.StockExchange + ":" + quote.Symbol, - description: quote.Name, - - lp: quote.LastTradePriceOnly, - ask: quote.AskRealtime, - bid: quote.BidRealtime, - - open_price: quote.Open, - high_price: quote.DaysHigh, - low_price: quote.DaysLow, - prev_close_price: quote.PreviousClose, - volume: quote.Volume, - } - }); - }); - return result; -} - -function proxyRequest(controller, options, response) { - controller.request(options, function(res) { - var result = ''; - - res.on('data', function (chunk) { - result += chunk; - }); - - res.on('end', function () { - if (res.statusCode !== 200) { - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify({ s: 'error', errmsg: 'Failed to get news' })); - response.end(); - return; - } - response.writeHead(200, defaultResponseHeader); - response.write(result); - response.end(); - }); - }).end(); -} - -function RequestProcessor(action, query, response) { - - this.sendError = function(error, response) { - response.writeHead(200, defaultResponseHeader); - response.write("{\"s\":\"error\",\"errmsg\":\"" + error + "\"}"); - response.end(); - - console.log(error); - }; - - - this.sendConfig = function(response) { - - var config = { - supports_search: true, - supports_group_request: false, - supports_marks: true, - supports_timescale_marks: true, - supports_time: true, - exchanges: [ - {value: "", name: "All Exchanges", desc: ""}, - {value: "NasdaqNM", name: "NasdaqNM", desc: "NasdaqNM"}, - {value: "NYSE", name: "NYSE", desc: "NYSE"}, - {value: "NCM", name: "NCM", desc: "NCM"}, - {value: "NGM", name: "NGM", desc: "NGM"}, - ], - symbolsTypes: [ - {name: "All types", value: ""}, - {name: "Stock", value: "stock"}, - {name: "Index", value: "index"} - ], - supportedResolutions: [ "D", "2D", "3D", "W", "3W", "M", '6M' ] - }; - - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(config)); - response.end(); - }; - - - this.sendMarks = function(response) { - var now = new Date(); - now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; - var day = 60 * 60 * 24; - - var marks = { - id: [0, 1, 2, 3, 4, 5], - time: [now, now - day * 4, now - day * 7, now - day * 7, now - day * 15, now - day * 30], - color: ["red", "blue", "green", "red", "blue", "green"], - text: ["Today", "4 days back", "7 days back + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", "7 days back once again", "15 days back", "30 days back"], - label: ["A", "B", "CORE", "D", "EURO", "F"], - labelFontColor: ["white", "white", "red", "#FFFFFF", "white", "#000"], - minSize: [14, 28, 7, 40, 7, 14] - }; - - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(marks)); - response.end(); - }; - - this.sendTime = function(response) { - var now = new Date(); - response.writeHead(200, defaultResponseHeader); - response.write(Math.floor(now / 1000) + ''); - response.end(); - }; - - this.sendTimescaleMarks = function(response) { - var now = new Date(); - now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; - var day = 60 * 60 * 24; - - var marks = [ - {id: "tsm1", time: now - day * 0, color: "red", label: "A", tooltip: ""}, - {id: "tsm2", time: now - day * 4, color: "blue", label: "D", tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()]}, - {id: "tsm3", time: now - day * 7, color: "green", label: "D", tooltip: ["Dividends: $3.46", "Date: " + new Date((now - day * 7) * 1000).toDateString()]}, - {id: "tsm4", time: now - day * 15, color: "#999999", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"]}, - {id: "tsm7", time: now - day * 30, color: "red", label: "E", tooltip: ["Earnings: $5.40", "Estimate: $5.00"]}, - ]; - - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(marks)); - response.end(); - }; - - - this.sendSymbolSearchResults = function(query, type, exchange, maxRecords, response) { - if (!maxRecords) { - throw "wrong_query"; - } - - var result = symbolsDatabase.search(query, type, exchange, maxRecords); - - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(result)); - response.end(); - }; - - this.sendSymbolInfo = function(symbolName, response) { - var symbolInfo = symbolsDatabase.symbolInfo(symbolName); - - if (!symbolInfo) { - throw "unknown_symbol " + symbolName; - } - - var info = { - "name": symbolInfo.name, - "exchange-traded": symbolInfo.exchange, - "exchange-listed": symbolInfo.exchange, - "timezone": "America/New_York", - "minmov": 1, - "minmov2": 0, - "pointvalue": 1, - "session": "0930-1630", - "has_intraday": false, - "has_no_volume": symbolInfo.type !== "stock", - "description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name, - "type": symbolInfo.type, - "supported_resolutions" : ["D","2D","3D","W","3W","M","6M"], - "pricescale": 100, - "ticker": symbolInfo.name.toUpperCase() - }; - - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(info)); - response.end(); - }; - - this.sendSymbolHistory = function(symbol, startDateTimestamp, endDateTimestamp, resolution, response) { - function dateToYMD(date) { - var obj = new Date(date * 1000); - var year = obj.getFullYear(); - var month = obj.getMonth() + 1; - var day = obj.getDate(); - return year + "-" + month + "-" + day; - } - - function sendResult(content) { - var header = createDefaultHeader(); - header["Content-Length"] = content.length; - response.writeHead(200, header); - response.write(content, null, function() { - response.end(); - }); - } - - var from = '1970-01-01'; // dateToYMD(startDateTimestamp); always return all data to reduce number of requests to quandl - var to = dateToYMD(endDateTimestamp); - - var key = symbol + "|" + from + "|" + to; - - if (quandlCache[key]) { - console.log("Return QUANDL result from cache: " + key); - sendResult(quandlCache[key]); - return; - } - - var address = "/api/v3/datatables/WIKI/PRICES.json" + - "?api_key=" + process.env.QUANDL_API_KEY + // you should create a free account on quandl.com to get this key - "&ticker=" + symbol + - "&date.gte=" + from + - "&date.lte=" + to; - - console.log("Sending request to quandl for symbol " + symbol + ". url=" + address); - - httpGet("www.quandl.com", address, function(result) { - if (response.finished) { - // we can be here if error happened on socket disconnect - return; - } - var content = JSON.stringify(convertQuandlHistoryToUDFFormat(result)); - quandlCache[key] = content; - sendResult(content); - }); - }; - - this.sendQuotes = function(tickersString, response) { - var tickersMap = {}; // maps YQL symbol to ticker - - var tickers = tickersString.split(","); - [].concat(tickers).forEach(function(ticker) { - var yqlSymbol = ticker.replace(/.*:(.*)/, "$1"); - tickersMap[yqlSymbol] = ticker; - }); - - var yql = "select * from yahoo.finance.quotes where symbol in ('" + Object.keys(tickersMap).join("','") + "')"; - console.log("Quotes query: " + yql); - - var options = { - host: "query.yahooapis.com", - path: "/v1/public/yql?q=" + encodeURIComponent(yql) + - "&format=json" + - "&env=store://datatables.org/alltableswithkeys" - }; - // for debug purposes - // console.log(options.host + options.path); - - http.request(options, function(res) { - var result = ''; - - res.on('data', function (chunk) { - result += chunk; - }); - - res.on('end', function () { - if (res.statusCode !== 200) { - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify({ s: 'error', errmsg: 'Yahoo fails' })); - response.end(); - return; - } - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(convertYahooQuotesToUDFFormat( - tickersMap, JSON.parse(result)))); - response.end(); - }); - }).end(); - }; - - this.sendNews = function(symbol, response) { - var options = { - host: "feeds.finance.yahoo.com", - path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US" - }; - - proxyRequest(https, options, response); - }; - - this.sendFuturesmag = function(response) { - var options = { - host: "www.futuresmag.com", - path: "/rss/all" - }; - - proxyRequest(http, options, response); - }; - - try - { - if (action === "/config") { - this.sendConfig(response); - } - else if (action === "/symbols" && !!query["symbol"]) { - this.sendSymbolInfo(query["symbol"], response); - } - else if (action === "/search") { - this.sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); - } - else if (action === "/history") { - this.sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response); - } - else if (action === "/quotes") { - this.sendQuotes(query["symbols"], response); - } - else if (action === "/marks") { - this.sendMarks(response); - } - else if (action === "/time") { - this.sendTime(response); - } - else if (action === "/timescale_marks") { - this.sendTimescaleMarks(response); - } - else if (action === "/news") { - this.sendNews(query["symbol"], response); - } - else if (action === "/futuresmag") { - this.sendFuturesmag(response); - } - } - catch (error) { - this.sendError(error, response); - } -} + symbolsDatabase = require("./symbols_database"), + RequestProcessor = require("./request-processor").RequestProcessor; +var requestProcessor = new RequestProcessor(symbolsDatabase); // Usage: // /config @@ -484,7 +48,7 @@ getFreePort(function(port) { http.createServer(function(request, response) { var uri = url.parse(request.url, true); var action = uri.pathname; - return new RequestProcessor(action, uri.query, response); + return requestProcessor.processRequest(action, uri.query, response); }).listen(port); From 387a8cf81a9f61f2299e74139423580d08868075 Mon Sep 17 00:00:00 2001 From: Eugene Timokhov Date: Thu, 7 Sep 2017 13:03:09 +0300 Subject: [PATCH 23/59] Changed config format (without breaking changes) --- request-processor.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/request-processor.js b/request-processor.js index 04154bc..14ec697 100644 --- a/request-processor.js +++ b/request-processor.js @@ -248,7 +248,7 @@ RequestProcessor.prototype._sendConfig = function (response) { desc: "NGM" }, ], - symbolsTypes: [ + symbols_types: [ { name: "All types", value: "" @@ -262,7 +262,7 @@ RequestProcessor.prototype._sendConfig = function (response) { value: "index" } ], - supportedResolutions: ["D", "2D", "3D", "W", "3W", "M", '6M'] + supported_resolutions: ["D", "2D", "3D", "W", "3W", "M", '6M'] }; response.writeHead(200, defaultResponseHeader); @@ -547,4 +547,4 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { } }; -exports.RequestProcessor = RequestProcessor; \ No newline at end of file +exports.RequestProcessor = RequestProcessor; From 4df70a4f05a8a9c512a3455c5e011f9f43efddec Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Wed, 13 Sep 2017 11:44:17 +0300 Subject: [PATCH 24/59] add logs, return exact ranges --- request-processor.js | 85 ++++++++++++++++++++++++++++++++++++++------ 1 file changed, 75 insertions(+), 10 deletions(-) diff --git a/request-processor.js b/request-processor.js index 14ec697..d8bfce1 100644 --- a/request-processor.js +++ b/request-processor.js @@ -25,6 +25,10 @@ setInterval(function () { quandlCache = {}; }, quandlCacheCleanupTime); +function dateForLogs() { + return (new Date).toISOString() + ': '; +} + function createDefaultHeader() { return { "Content-Type": "text/plain", @@ -115,7 +119,8 @@ function convertQuandlHistoryToUDFFormat(data) { }); } catch (error) { - console.error(error); + var dataStr = typeof data === "string" ? data.slice(0, 100) : data; + console.error(dateForLogs() + error + ", failed to parse: " + dataStr); } return result; @@ -204,6 +209,41 @@ function RequestProcessor(symbolsDatabase) { this._symbolsDatabase = symbolsDatabase; } +function filterDataPeriod(data, fromSeconds, toSeconds) { + if (!data || !data.t || data.t.length === 0) { + return data; + } + + var fromIndex = null; + var toIndex = null; + var times = data.t; + for (var i = 0; i < times.length; i++) { + var time = times[i]; + if (fromIndex === null && time >= fromSeconds) { + fromIndex = i; + } + if (toIndex === null && time >= toSeconds) { + toIndex = time > toSeconds ? i - 1 : i; + } + if (fromIndex !== null && toIndex !== null) { + break; + } + } + + fromIndex = fromIndex || 0; + toIndex = toIndex ? toIndex + 1 : times.length; + + return { + t: data.t.slice(fromIndex, toIndex), + o: data.o.slice(fromIndex, toIndex), + h: data.h.slice(fromIndex, toIndex), + l: data.l.slice(fromIndex, toIndex), + c: data.c.slice(fromIndex, toIndex), + v: data.v.slice(fromIndex, toIndex), + s: data.s + }; +} + RequestProcessor.prototype._sendError = function (error, response) { response.writeHead(200, defaultResponseHeader); response.write("{\"s\":\"error\",\"errmsg\":\"" + error + "\"}"); @@ -395,7 +435,7 @@ RequestProcessor.prototype._sendSymbolInfo = function (symbolName, response) { RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimestamp, endDateTimestamp, resolution, response) { function dateToYMD(date) { - var obj = new Date(date * 1000); + var obj = new Date(date); var year = obj.getFullYear(); var month = obj.getMonth() + 1; var day = obj.getDate(); @@ -411,14 +451,31 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes }); } - var from = '1970-01-01'; // dateToYMD(startDateTimestamp); always return all data to reduce number of requests to quandl - var to = dateToYMD(endDateTimestamp); + function secondsToISO(sec) { + if (sec === null || sec === undefined) { + return 'n/a'; + } + return (new Date(sec * 1000).toISOString()); + } + + function logForData(data, key, isCached) { + var fromCacheTime = data && data.t ? data.t[0] : null; + var toCacheTime = data && data.t ? data.t[data.t.length - 1] : null; + console.log(dateForLogs() + "Return QUANDL result" + (isCached ? " from cache" : "") + ": " + key + ", from " + secondsToISO(fromCacheTime) + " to " + secondsToISO(toCacheTime)); + } + + console.log(dateForLogs() + "Got history request for " + symbol + ", " + resolution + " from " + secondsToISO(startDateTimestamp)+ " to " + secondsToISO(endDateTimestamp)); + + // always request all data to reduce number of requests to quandl + var from = '1970-01-01'; + var to = dateToYMD(Date.now()); var key = symbol + "|" + from + "|" + to; if (quandlCache[key]) { - console.log("Return QUANDL result from cache: " + key); - sendResult(quandlCache[key]); + var dataFromCache = filterDataPeriod(quandlCache[key], startDateTimestamp, endDateTimestamp); + logForData(dataFromCache, key, true); + sendResult(JSON.stringify(dataFromCache)); return; } @@ -428,16 +485,24 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes "&date.gte=" + from + "&date.lte=" + to; - console.log("Sending request to quandl for symbol " + symbol + ". url=" + address); + console.log(dateForLogs() + "Sending request to quandl " + key + ". url=" + address); httpGet("www.quandl.com", address, function (result) { if (response.finished) { // we can be here if error happened on socket disconnect return; } - var content = JSON.stringify(convertQuandlHistoryToUDFFormat(result)); - quandlCache[key] = content; - sendResult(content); + console.log(dateForLogs() + "Got response from quandl " + key + ". Try to parse."); + var data = convertQuandlHistoryToUDFFormat(result); + if (data.t.length !== 0) { + console.log(dateForLogs() + "Successfully parsed and put to cache " + data.t.length + " bars."); + quandlCache[key] = data; + } else { + console.warn(dateForLogs() + "Parsing returned empty result."); + } + var filteredData = filterDataPeriod(data, startDateTimestamp, endDateTimestamp); + logForData(filteredData, key, false); + sendResult(JSON.stringify(filteredData)); }); }; From f4c521296f9e2b4abb92937bba0b9da5b698a26a Mon Sep 17 00:00:00 2001 From: Eugene Timokhov Date: Fri, 22 Sep 2017 17:07:39 +0300 Subject: [PATCH 25/59] Change type of quote fields --- request-processor.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/request-processor.js b/request-processor.js index d8bfce1..15a7c4e 100644 --- a/request-processor.js +++ b/request-processor.js @@ -157,23 +157,23 @@ function convertYahooQuotesToUDFFormat(tickersMap, data) { s: "ok", n: ticker, v: { - ch: quote.ChangeRealtime || quote.Change, - chp: (quote.PercentChange || quote.ChangeinPercent) && (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1"), + ch: +(quote.ChangeRealtime || quote.Change), + chp: +((quote.PercentChange || quote.ChangeinPercent) && (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1")), short_name: quote.Symbol, exchange: quote.StockExchange, original_name: quote.StockExchange + ":" + quote.Symbol, description: quote.Name, - lp: quote.LastTradePriceOnly, - ask: quote.AskRealtime, - bid: quote.BidRealtime, + lp: +quote.LastTradePriceOnly, + ask: +quote.AskRealtime, + bid: +quote.BidRealtime, - open_price: quote.Open, - high_price: quote.DaysHigh, - low_price: quote.DaysLow, - prev_close_price: quote.PreviousClose, - volume: quote.Volume, + open_price: +quote.Open, + high_price: +quote.DaysHigh, + low_price: +quote.DaysLow, + prev_close_price: +quote.PreviousClose, + volume: +quote.Volume, } }); }); @@ -605,6 +605,10 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { } else if (action === "/futuresmag") { this._sendFuturesmag(response); + } else { + response.writeHead(200, defaultResponseHeader); + response.write('datafeed is ok'); + response.end(); } } catch (error) { From 5ce3d39271640cf3cc6cee439b1fe2fd7cb8bbbc Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Tue, 31 Oct 2017 11:53:40 +0300 Subject: [PATCH 26/59] FIXED: odd yahoo error No definition found for Table yahoo.finance.quotes --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index 15a7c4e..6e40f64 100644 --- a/request-processor.js +++ b/request-processor.js @@ -515,7 +515,7 @@ RequestProcessor.prototype._sendQuotes = function (tickersString, response) { tickersMap[yqlSymbol] = ticker; }); - var yql = "select * from yahoo.finance.quotes where symbol in ('" + Object.keys(tickersMap).join("','") + "')"; + var yql = "env 'store://datatables.org/alltableswithkeys'; select * from yahoo.finance.quotes where symbol in ('" + Object.keys(tickersMap).join("','") + "')"; console.log("Quotes query: " + yql); var options = { From 2df2d0a4763ce951374d5db04ea4c5c2d3917ec7 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Fri, 3 Nov 2017 12:33:31 +0300 Subject: [PATCH 27/59] Return quandl bar when yahoo is unavailable --- request-processor.js | 135 ++++++++++++++++++++++++++++++++----------- 1 file changed, 101 insertions(+), 34 deletions(-) diff --git a/request-processor.js b/request-processor.js index 6e40f64..4c692d0 100644 --- a/request-processor.js +++ b/request-processor.js @@ -19,6 +19,8 @@ var http = require("http"); var quandlCache = {}; var quandlCacheCleanupTime = 3 * 60 * 60 * 100; // 3 hours +var yahooFailedStateCacheTime = 3 * 60 * 60 * 100; // 3 hours; +var quandlMinimumDate = '1970-01-01'; // this cache is intended to reduce number of requests to Quandl setInterval(function () { @@ -26,17 +28,27 @@ setInterval(function () { }, quandlCacheCleanupTime); function dateForLogs() { - return (new Date).toISOString() + ': '; + return (new Date()).toISOString() + ': '; } -function createDefaultHeader() { - return { - "Content-Type": "text/plain", - 'Access-Control-Allow-Origin': '*' - }; +var defaultResponseHeader = { + "Content-Type": "text/plain", + 'Access-Control-Allow-Origin': '*' +}; + +function sendJsonResponse(response, jsonData) { + response.writeHead(200, defaultResponseHeader); + response.write(JSON.stringify(jsonData)); + response.end(); } -var defaultResponseHeader = createDefaultHeader(); +function dateToYMD(date) { + var obj = new Date(date); + var year = obj.getFullYear(); + var month = obj.getMonth() + 1; + var day = obj.getDate(); + return year + "-" + month + "-" + day; +} function httpGet(datafeedHost, path, callback) { var options = { @@ -140,6 +152,7 @@ function convertYahooQuotesToUDFFormat(tickersMap, data) { s: "ok", d: [] }; + [].concat(data.query.results.quote).forEach(function (quote) { var ticker = tickersMap[quote.symbol]; @@ -207,6 +220,7 @@ function proxyRequest(controller, options, response) { function RequestProcessor(symbolsDatabase) { this._symbolsDatabase = symbolsDatabase; + this._failedYahooTime = {}; } function filterDataPeriod(data, fromSeconds, toSeconds) { @@ -346,7 +360,7 @@ RequestProcessor.prototype._sendTimescaleMarks = function (response) { var marks = [ { id: "tsm1", - time: now - day * 0, + time: now, color: "red", label: "A", tooltip: "" @@ -434,16 +448,8 @@ RequestProcessor.prototype._sendSymbolInfo = function (symbolName, response) { }; RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimestamp, endDateTimestamp, resolution, response) { - function dateToYMD(date) { - var obj = new Date(date); - var year = obj.getFullYear(); - var month = obj.getMonth() + 1; - var day = obj.getDate(); - return year + "-" + month + "-" + day; - } - function sendResult(content) { - var header = createDefaultHeader(); + var header = Object.assign({}, defaultResponseHeader); header["Content-Length"] = content.length; response.writeHead(200, header); response.write(content, null, function () { @@ -467,7 +473,7 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes console.log(dateForLogs() + "Got history request for " + symbol + ", " + resolution + " from " + secondsToISO(startDateTimestamp)+ " to " + secondsToISO(endDateTimestamp)); // always request all data to reduce number of requests to quandl - var from = '1970-01-01'; + var from = quandlMinimumDate; var to = dateToYMD(Date.now()); var key = symbol + "|" + from + "|" + to; @@ -506,6 +512,61 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes }); }; +RequestProcessor.prototype._quotesQuandlWorkaround = function (tickersMap) { + var from = quandlMinimumDate; + var to = dateToYMD(Date.now()); + + var result = { + s: "ok", + d: [], + source: 'Quandl', + }; + + Object.keys(tickersMap).forEach(function(symbol) { + var key = symbol + "|" + from + "|" + to; + var ticker = tickersMap[symbol]; + + var data = quandlCache[key]; + var length = data === undefined ? 0 : data.c.length; + + if (length > 0) { + var lastBar = { + o: data.o[length - 1], + h: data.o[length - 1], + l: data.o[length - 1], + c: data.o[length - 1], + v: data.o[length - 1], + }; + + result.d.push({ + s: "ok", + n: ticker, + v: { + ch: 0, + chp: 0, + + short_name: symbol, + exchange: '', + original_name: ticker, + description: ticker, + + lp: lastBar.c, + ask: lastBar.c, + bid: lastBar.c, + + open_price: lastBar.o, + high_price: lastBar.h, + low_price: lastBar.l, + prev_close_price: length > 1 ? data.c[length - 2] : lastBar.o, + volume: lastBar.v, + } + }); + } + }); + + return result; +}; + RequestProcessor.prototype._sendQuotes = function (tickersString, response) { var tickersMap = {}; // maps YQL symbol to ticker @@ -515,6 +576,14 @@ RequestProcessor.prototype._sendQuotes = function (tickersString, response) { tickersMap[yqlSymbol] = ticker; }); + if (this._failedYahooTime[tickersString] && Date.now() - this._failedYahooTime[tickersString] < yahooFailedStateCacheTime) { + sendJsonResponse(response, this._quotesQuandlWorkaround(tickersMap)); + console.log("Quotes request : " + tickersString + ' processed from quandl cache'); + return; + } + + var that = this; + var yql = "env 'store://datatables.org/alltableswithkeys'; select * from yahoo.finance.quotes where symbol in ('" + Object.keys(tickersMap).join("','") + "')"; console.log("Quotes query: " + yql); @@ -535,19 +604,21 @@ RequestProcessor.prototype._sendQuotes = function (tickersString, response) { }); res.on('end', function () { - if (res.statusCode !== 200) { - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify({ - s: 'error', - errmsg: 'Yahoo fails' - })); - response.end(); - return; + var jsonResponse = { s: 'error' }; + + if (res.statusCode === 200) { + jsonResponse = convertYahooQuotesToUDFFormat(tickersMap, JSON.parse(result)); + } else { + console.error('Yahoo Fails with code ' + res.statusCode); } - response.writeHead(200, defaultResponseHeader); - response.write(JSON.stringify(convertYahooQuotesToUDFFormat( - tickersMap, JSON.parse(result)))); - response.end(); + + if (jsonResponse.s === 'error') { + that._failedYahooTime[tickersString] = Date.now(); + jsonResponse = that._quotesQuandlWorkaround(tickersMap); + console.log("Quotes request : " + tickersString + ' processed from quandl'); + } + + sendJsonResponse(response, jsonResponse); }); }).end(); }; @@ -570,10 +641,6 @@ RequestProcessor.prototype._sendFuturesmag = function (response) { proxyRequest(http, options, response); }; -RequestProcessor.prototype._defaultResponseHeader = function() { - return defaultResponseHeader; -}; - RequestProcessor.prototype.processRequest = function (action, query, response) { try { if (action === "/config") { From 9d0e2f1e00bd8cd232d29bc0d8d55e73fc84120d Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Mon, 12 Feb 2018 16:44:46 +0300 Subject: [PATCH 28/59] send no_data --- request-processor.js | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/request-processor.js b/request-processor.js index 4c692d0..94586cc 100644 --- a/request-processor.js +++ b/request-processor.js @@ -224,7 +224,7 @@ function RequestProcessor(symbolsDatabase) { } function filterDataPeriod(data, fromSeconds, toSeconds) { - if (!data || !data.t || data.t.length === 0) { + if (!data || !data.t) { return data; } @@ -247,6 +247,12 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { fromIndex = fromIndex || 0; toIndex = toIndex ? toIndex + 1 : times.length; + var s = data.s; + + if (toSeconds < times[0]) { + s = 'no_data'; + } + return { t: data.t.slice(fromIndex, toIndex), o: data.o.slice(fromIndex, toIndex), @@ -254,7 +260,7 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { l: data.l.slice(fromIndex, toIndex), c: data.c.slice(fromIndex, toIndex), v: data.v.slice(fromIndex, toIndex), - s: data.s + s: s }; } From 2bd18dc8eb5d0500b907c2d3c03b721086bd4279 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Wed, 28 Mar 2018 11:53:06 +0300 Subject: [PATCH 29/59] Use set of quandl keys --- request-processor.js | 75 ++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 17 deletions(-) diff --git a/request-processor.js b/request-processor.js index 94586cc..5735192 100644 --- a/request-processor.js +++ b/request-processor.js @@ -13,6 +13,8 @@ "use strict"; +var version = '2.0.0'; + var https = require("https"); var http = require("http"); @@ -50,6 +52,37 @@ function dateToYMD(date) { return year + "-" + month + "-" + day; } +var quandlKeys = process.env.QUANDL_API_KEY.split(','); // you should create a free account on quandl.com to get this key, you can set some keys concatenated with a comma +var invalidQuandlKeys = []; + +function getValidQuandlKey() { + for (var i = 0; i < quandlKeys.length; i++) { + var key = quandlKeys[i]; + if (invalidQuandlKeys.indexOf(key) === -1) { + return key; + } + } + return null; +} + +function markQuandlKeyAsInvalid(key) { + if (invalidQuandlKeys.indexOf(key) !== -1) { + return; + } + + invalidQuandlKeys.push(key); + + setTimeout(function() { + invalidQuandlKeys.shift(); + }, quandlCacheCleanupTime); +} + +function sendError(error, response) { + response.writeHead(200, defaultResponseHeader); + response.write("{\"s\":\"error\",\"errmsg\":\"" + error + "\"}"); + response.end(); +} + function httpGet(datafeedHost, path, callback) { var options = { host: datafeedHost, @@ -65,7 +98,7 @@ function httpGet(datafeedHost, path, callback) { response.on('end', function () { if (response.statusCode !== 200) { - callback(''); + callback(response.statusMessage || ''); return; } @@ -85,7 +118,7 @@ function httpGet(datafeedHost, path, callback) { req.on('error', function (e) { console.log('Problem with request: ' + e.message); - callback(''); + callback(e.message); }); req.end(); @@ -131,8 +164,7 @@ function convertQuandlHistoryToUDFFormat(data) { }); } catch (error) { - var dataStr = typeof data === "string" ? data.slice(0, 100) : data; - console.error(dateForLogs() + error + ", failed to parse: " + dataStr); + return null; } return result; @@ -264,15 +296,6 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { }; } -RequestProcessor.prototype._sendError = function (error, response) { - response.writeHead(200, defaultResponseHeader); - response.write("{\"s\":\"error\",\"errmsg\":\"" + error + "\"}"); - response.end(); - - console.log(error); -}; - - RequestProcessor.prototype._sendConfig = function (response) { var config = { @@ -491,8 +514,16 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes return; } + var quandlKey = getValidQuandlKey(); + + if (quandlKey === null) { + console.log(dateForLogs() + "No valid quandl key available"); + sendError('No API Key', response); + return; + } + var address = "/api/v3/datatables/WIKI/PRICES.json" + - "?api_key=" + process.env.QUANDL_API_KEY + // you should create a free account on quandl.com to get this key + "?api_key=" + quandlKey + // you should create a free account on quandl.com to get this key "&ticker=" + symbol + "&date.gte=" + from + "&date.lte=" + to; @@ -506,12 +537,21 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes } console.log(dateForLogs() + "Got response from quandl " + key + ". Try to parse."); var data = convertQuandlHistoryToUDFFormat(result); + if (data === null) { + var dataStr = typeof result === "string" ? result.slice(0, 100) : result; + console.error(dateForLogs() + " failed to parse: " + dataStr); + markQuandlKeyAsInvalid(quandlKey); + sendError("Invalid quandl response", response); + return; + } + if (data.t.length !== 0) { console.log(dateForLogs() + "Successfully parsed and put to cache " + data.t.length + " bars."); quandlCache[key] = data; } else { - console.warn(dateForLogs() + "Parsing returned empty result."); + console.log(dateForLogs() + "Parsing returned empty result."); } + var filteredData = filterDataPeriod(data, startDateTimestamp, endDateTimestamp); logForData(filteredData, key, false); sendResult(JSON.stringify(filteredData)); @@ -680,12 +720,13 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendFuturesmag(response); } else { response.writeHead(200, defaultResponseHeader); - response.write('datafeed is ok'); + response.write('Datafeed version is ' + version + '. Valid keys count is ' + String(quandlKeys.length - invalidQuandlKeys.length)); response.end(); } } catch (error) { - this._sendError(error, response); + sendError(error, response); + console.error('Exception: ' + error); } }; From bcdd3323aad9a93bf86c31bb91b649b799373b58 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Wed, 28 Mar 2018 13:48:18 +0300 Subject: [PATCH 30/59] do not invalidate keys on socket errors --- request-processor.js | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/request-processor.js b/request-processor.js index 5735192..2229d3f 100644 --- a/request-processor.js +++ b/request-processor.js @@ -72,8 +72,10 @@ function markQuandlKeyAsInvalid(key) { invalidQuandlKeys.push(key); + console.warn('Quandl key invalidated ' + key); + setTimeout(function() { - invalidQuandlKeys.shift(); + console.log("Quandl key restored: " + invalidQuandlKeys.shift()); }, quandlCacheCleanupTime); } @@ -98,11 +100,11 @@ function httpGet(datafeedHost, path, callback) { response.on('end', function () { if (response.statusCode !== 200) { - callback(response.statusMessage || ''); + callback({ status: 'ERR_STATUS_CODE', errmsg: response.statusMessage || '' }); return; } - callback(result); + callback({ status: 'ok', data: result }); }); } @@ -117,8 +119,7 @@ function httpGet(datafeedHost, path, callback) { }); req.on('error', function (e) { - console.log('Problem with request: ' + e.message); - callback(e.message); + callback({ status: 'ERR_SOCKET', errmsg: e.message || '' }); }); req.end(); @@ -518,7 +519,7 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes if (quandlKey === null) { console.log(dateForLogs() + "No valid quandl key available"); - sendError('No API Key', response); + sendError('No valid API Keys available', response); return; } @@ -535,12 +536,25 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes // we can be here if error happened on socket disconnect return; } + + if (result.status !== 'ok') { + if (result.status === 'ERR_SOCKET') { + console.log('Socket problem with request: ' + result.errmsg); + sendError("Socket problem with request " + result.errmsg, response); + return; + } + + console.error(dateForLogs() + "Error response from quandl for key " + key + ". Message: " + result.errmsg); + markQuandlKeyAsInvalid(quandlKey); + sendError("Error quandl response " + result.errmsg, response); + return; + } + console.log(dateForLogs() + "Got response from quandl " + key + ". Try to parse."); - var data = convertQuandlHistoryToUDFFormat(result); + var data = convertQuandlHistoryToUDFFormat(result.data); if (data === null) { var dataStr = typeof result === "string" ? result.slice(0, 100) : result; console.error(dateForLogs() + " failed to parse: " + dataStr); - markQuandlKeyAsInvalid(quandlKey); sendError("Invalid quandl response", response); return; } @@ -720,7 +734,10 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendFuturesmag(response); } else { response.writeHead(200, defaultResponseHeader); - response.write('Datafeed version is ' + version + '. Valid keys count is ' + String(quandlKeys.length - invalidQuandlKeys.length)); + response.write('Datafeed version is ' + version + + '\nValid keys count is ' + String(quandlKeys.length - invalidQuandlKeys.length) + + '\nCurrent key is ' + (getValidQuandlKey() || '').slice(0, 3) + + (invalidQuandlKeys.length !== 0 ? '\nInvalid keys are ' + invalidQuandlKeys.reduce(function(prev, cur) { return prev + cur.slice(0, 3) + ','; }, '') : '')); response.end(); } } From f4cc53dbe4609ebc225885849455948954c7694a Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 29 Mar 2018 15:20:10 +0300 Subject: [PATCH 31/59] fix cache invalidation time, format logs --- request-processor.js | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/request-processor.js b/request-processor.js index 2229d3f..a17c485 100644 --- a/request-processor.js +++ b/request-processor.js @@ -20,13 +20,15 @@ var http = require("http"); var quandlCache = {}; -var quandlCacheCleanupTime = 3 * 60 * 60 * 100; // 3 hours -var yahooFailedStateCacheTime = 3 * 60 * 60 * 100; // 3 hours; +var quandlCacheCleanupTime = 3 * 60 * 60 * 1000; // 3 hours +var quandlKeysValidateTime = 15 * 60 * 1000; // 15 minutes +var yahooFailedStateCacheTime = 3 * 60 * 60 * 1000; // 3 hours; var quandlMinimumDate = '1970-01-01'; // this cache is intended to reduce number of requests to Quandl setInterval(function () { quandlCache = {}; + console.warn(dateForLogs() + 'Quandl cache invalidated'); }, quandlCacheCleanupTime); function dateForLogs() { @@ -72,11 +74,11 @@ function markQuandlKeyAsInvalid(key) { invalidQuandlKeys.push(key); - console.warn('Quandl key invalidated ' + key); + console.warn(dateForLogs() + 'Quandl key invalidated ' + key); setTimeout(function() { - console.log("Quandl key restored: " + invalidQuandlKeys.shift()); - }, quandlCacheCleanupTime); + console.log(dateForLogs() + "Quandl key restored: " + invalidQuandlKeys.shift()); + }, quandlKeysValidateTime); } function sendError(error, response) { @@ -113,7 +115,7 @@ function httpGet(datafeedHost, path, callback) { req.on('socket', function (socket) { socket.setTimeout(5000); socket.on('timeout', function () { - console.log('timeout'); + console.log(dateForLogs() + 'timeout'); req.abort(); }); }); @@ -174,7 +176,7 @@ function convertQuandlHistoryToUDFFormat(data) { function convertYahooQuotesToUDFFormat(tickersMap, data) { if (!data.query || !data.query.results) { var errmsg = "ERROR: empty quotes response: " + JSON.stringify(data); - console.log(errmsg); + console.log(dateForLogs() + errmsg); return { s: "error", errmsg: errmsg From 97d60d7945a4b069da61a67880829278db03365b Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Thu, 29 Mar 2018 15:23:42 +0300 Subject: [PATCH 32/59] increase version --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index a17c485..0fda129 100644 --- a/request-processor.js +++ b/request-processor.js @@ -13,7 +13,7 @@ "use strict"; -var version = '2.0.0'; +var version = '2.0.1'; var https = require("https"); var http = require("http"); From a6b6aac4ebca776105f974a237d0d2b3bd959e3c Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Tue, 10 Apr 2018 11:54:24 +0300 Subject: [PATCH 33/59] FIXED: too big responses overloading CPU --- request-processor.js | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/request-processor.js b/request-processor.js index 0fda129..035dfe2 100644 --- a/request-processor.js +++ b/request-processor.js @@ -263,6 +263,13 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { return data; } + if (data.t[data.t.length - 1] < fromSeconds) { + return { + s: 'no_data', + nextTime: data.t[data.t.length - 1] + }; + } + var fromIndex = null; var toIndex = null; var times = data.t; @@ -288,6 +295,8 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { s = 'no_data'; } + toIndex = Math.min(fromIndex + 1000, toIndex); // do not send more than 1000 bars for server capacity reasons + return { t: data.t.slice(fromIndex, toIndex), o: data.o.slice(fromIndex, toIndex), From 75f16ae9d08eae26603adad06072eb1f484eb107 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Tue, 10 Apr 2018 12:05:09 +0300 Subject: [PATCH 34/59] update version --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index 035dfe2..4a7c2d0 100644 --- a/request-processor.js +++ b/request-processor.js @@ -13,7 +13,7 @@ "use strict"; -var version = '2.0.1'; +var version = '2.0.2'; var https = require("https"); var http = require("http"); From 7296fe0c87a10686cfa7b531d39377abe330a15d Mon Sep 17 00:00:00 2001 From: Eugene Timokhov Date: Tue, 21 Aug 2018 22:27:55 +0300 Subject: [PATCH 35/59] Sort data by time in asc order --- request-processor.js | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/request-processor.js b/request-processor.js index 4a7c2d0..1546faa 100644 --- a/request-processor.js +++ b/request-processor.js @@ -157,14 +157,18 @@ function convertQuandlHistoryToUDFFormat(data) { var datatable = json.datatable; var idx = columnIndices(datatable.columns); - datatable.data.forEach(function (row) { - result.t.push(parseDate(row[idx.date]) / 1000); - result.o.push(row[idx.open]); - result.h.push(row[idx.high]); - result.l.push(row[idx.low]); - result.c.push(row[idx.close]); - result.v.push(row[idx.volume]); - }); + datatable.data + .sort(function (row1, row2) { + return parseDate(row1[idx.date]) - parseDate(row2[idx.date]); + }) + .forEach(function (row) { + result.t.push(parseDate(row[idx.date]) / 1000); + result.o.push(row[idx.open]); + result.h.push(row[idx.high]); + result.l.push(row[idx.low]); + result.c.push(row[idx.close]); + result.v.push(row[idx.volume]); + }); } catch (error) { return null; From 397947e7191273dd536f7008d8b19ba3ba0fdd7b Mon Sep 17 00:00:00 2001 From: Eugene Timokhov Date: Tue, 21 Aug 2018 22:34:32 +0300 Subject: [PATCH 36/59] update version --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index 1546faa..4c54285 100644 --- a/request-processor.js +++ b/request-processor.js @@ -13,7 +13,7 @@ "use strict"; -var version = '2.0.2'; +var version = '2.0.3'; var https = require("https"); var http = require("http"); From dd21bb590bf4965233ca3c071902f132a2ae0211 Mon Sep 17 00:00:00 2001 From: Evgeniy Timokhov Date: Fri, 4 Jan 2019 10:26:30 +0300 Subject: [PATCH 37/59] Removed requesting quotes from yahoo --- request-processor.js | 104 +------------------------------------------ 1 file changed, 2 insertions(+), 102 deletions(-) diff --git a/request-processor.js b/request-processor.js index 4c54285..a943f03 100644 --- a/request-processor.js +++ b/request-processor.js @@ -22,7 +22,6 @@ var quandlCache = {}; var quandlCacheCleanupTime = 3 * 60 * 60 * 1000; // 3 hours var quandlKeysValidateTime = 15 * 60 * 1000; // 15 minutes -var yahooFailedStateCacheTime = 3 * 60 * 60 * 1000; // 3 hours; var quandlMinimumDate = '1970-01-01'; // this cache is intended to reduce number of requests to Quandl @@ -177,61 +176,6 @@ function convertQuandlHistoryToUDFFormat(data) { return result; } -function convertYahooQuotesToUDFFormat(tickersMap, data) { - if (!data.query || !data.query.results) { - var errmsg = "ERROR: empty quotes response: " + JSON.stringify(data); - console.log(dateForLogs() + errmsg); - return { - s: "error", - errmsg: errmsg - }; - } - - var result = { - s: "ok", - d: [] - }; - - [].concat(data.query.results.quote).forEach(function (quote) { - var ticker = tickersMap[quote.symbol]; - - // this field is an error token - if (quote["ErrorIndicationreturnedforsymbolchangedinvalid"] || !quote.StockExchange) { - result.d.push({ - s: "error", - n: ticker, - v: {} - }); - return; - } - - result.d.push({ - s: "ok", - n: ticker, - v: { - ch: +(quote.ChangeRealtime || quote.Change), - chp: +((quote.PercentChange || quote.ChangeinPercent) && (quote.PercentChange || quote.ChangeinPercent).replace(/[+-]?(.*)%/, "$1")), - - short_name: quote.Symbol, - exchange: quote.StockExchange, - original_name: quote.StockExchange + ":" + quote.Symbol, - description: quote.Name, - - lp: +quote.LastTradePriceOnly, - ask: +quote.AskRealtime, - bid: +quote.BidRealtime, - - open_price: +quote.Open, - high_price: +quote.DaysHigh, - low_price: +quote.DaysLow, - prev_close_price: +quote.PreviousClose, - volume: +quote.Volume, - } - }); - }); - return result; -} - function proxyRequest(controller, options, response) { controller.request(options, function (res) { var result = ''; @@ -259,7 +203,6 @@ function proxyRequest(controller, options, response) { function RequestProcessor(symbolsDatabase) { this._symbolsDatabase = symbolsDatabase; - this._failedYahooTime = {}; } function filterDataPeriod(data, fromSeconds, toSeconds) { @@ -651,51 +594,8 @@ RequestProcessor.prototype._sendQuotes = function (tickersString, response) { tickersMap[yqlSymbol] = ticker; }); - if (this._failedYahooTime[tickersString] && Date.now() - this._failedYahooTime[tickersString] < yahooFailedStateCacheTime) { - sendJsonResponse(response, this._quotesQuandlWorkaround(tickersMap)); - console.log("Quotes request : " + tickersString + ' processed from quandl cache'); - return; - } - - var that = this; - - var yql = "env 'store://datatables.org/alltableswithkeys'; select * from yahoo.finance.quotes where symbol in ('" + Object.keys(tickersMap).join("','") + "')"; - console.log("Quotes query: " + yql); - - var options = { - host: "query.yahooapis.com", - path: "/v1/public/yql?q=" + encodeURIComponent(yql) + - "&format=json" + - "&env=store://datatables.org/alltableswithkeys" - }; - // for debug purposes - // console.log(options.host + options.path); - - http.request(options, function (res) { - var result = ''; - - res.on('data', function (chunk) { - result += chunk; - }); - - res.on('end', function () { - var jsonResponse = { s: 'error' }; - - if (res.statusCode === 200) { - jsonResponse = convertYahooQuotesToUDFFormat(tickersMap, JSON.parse(result)); - } else { - console.error('Yahoo Fails with code ' + res.statusCode); - } - - if (jsonResponse.s === 'error') { - that._failedYahooTime[tickersString] = Date.now(); - jsonResponse = that._quotesQuandlWorkaround(tickersMap); - console.log("Quotes request : " + tickersString + ' processed from quandl'); - } - - sendJsonResponse(response, jsonResponse); - }); - }).end(); + sendJsonResponse(response, this._quotesQuandlWorkaround(tickersMap)); + console.log("Quotes request : " + tickersString + ' processed from quandl cache'); }; RequestProcessor.prototype._sendNews = function (symbol, response) { From cbd753972483af4ff06b68d7f68e4d503cb84afc Mon Sep 17 00:00:00 2001 From: Evgeniy Timokhov Date: Fri, 4 Jan 2019 10:28:05 +0300 Subject: [PATCH 38/59] Updated version --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index a943f03..c7bff25 100644 --- a/request-processor.js +++ b/request-processor.js @@ -13,7 +13,7 @@ "use strict"; -var version = '2.0.3'; +var version = '2.0.4'; var https = require("https"); var http = require("http"); From d9daa9bf273c360dd816c47e14b559fd305a1e58 Mon Sep 17 00:00:00 2001 From: Kirill Chetverikov Date: Mon, 21 Jan 2019 20:01:00 +0300 Subject: [PATCH 39/59] Changed ticker calculation --- request-processor.js | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/request-processor.js b/request-processor.js index c7bff25..15cffea 100644 --- a/request-processor.js +++ b/request-processor.js @@ -255,6 +255,15 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { }; } +function removeExchangeFromSymbol(symbol) { + var result = symbol.split(':'); + return result[result.length - 1]; +} + +function removeExchangeFromSymbols(symbols) { + return symbols.split(',').map((symbol) => removeExchangeFromSymbol(symbol)).join(','); +} + RequestProcessor.prototype._sendConfig = function (response) { var config = { @@ -410,8 +419,8 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { return { "name": symbolInfo.name, - "exchange-traded": symbolInfo.exchange, - "exchange-listed": symbolInfo.exchange, + "exchange-traded": symbolInfo.exchange.toUpperCase(), + "exchange-listed": symbolInfo.exchange.toUpperCase(), "timezone": "America/New_York", "minmov": 1, "minmov2": 0, @@ -423,7 +432,7 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { "type": symbolInfo.type, "supported_resolutions": ["D", "2D", "3D", "W", "3W", "M", "6M"], "pricescale": 100, - "ticker": symbolInfo.name.toUpperCase() + "ticker": (symbolInfo.exchange + ':' + symbolInfo.name).toUpperCase(), }; }; @@ -622,16 +631,16 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendConfig(response); } else if (action === "/symbols" && !!query["symbol"]) { - this._sendSymbolInfo(query["symbol"], response); + this._sendSymbolInfo(removeExchangeFromSymbol(query["symbol"]), response); } else if (action === "/search") { this._sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); } else if (action === "/history") { - this._sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response); + this._sendSymbolHistory(removeExchangeFromSymbol(query["symbol"]), query["from"], query["to"], query["resolution"].toLowerCase(), response); } else if (action === "/quotes") { - this._sendQuotes(query["symbols"], response); + this._sendQuotes(removeExchangeFromSymbols(query["symbols"]), response); } else if (action === "/marks") { this._sendMarks(response); @@ -643,7 +652,7 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendTimescaleMarks(response); } else if (action === "/news") { - this._sendNews(query["symbol"], response); + this._sendNews(removeExchangeFromSymbol(query["symbol"]), response); } else if (action === "/futuresmag") { this._sendFuturesmag(response); From 12695f137a070ef0a71e0fe540020b44e8f9b2c3 Mon Sep 17 00:00:00 2001 From: Evgeniy Timokhov Date: Wed, 23 Jan 2019 15:45:33 +0300 Subject: [PATCH 40/59] Revert "Changed ticker calculation" This reverts commit d9daa9bf273c360dd816c47e14b559fd305a1e58. --- request-processor.js | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/request-processor.js b/request-processor.js index 15cffea..c7bff25 100644 --- a/request-processor.js +++ b/request-processor.js @@ -255,15 +255,6 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { }; } -function removeExchangeFromSymbol(symbol) { - var result = symbol.split(':'); - return result[result.length - 1]; -} - -function removeExchangeFromSymbols(symbols) { - return symbols.split(',').map((symbol) => removeExchangeFromSymbol(symbol)).join(','); -} - RequestProcessor.prototype._sendConfig = function (response) { var config = { @@ -419,8 +410,8 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { return { "name": symbolInfo.name, - "exchange-traded": symbolInfo.exchange.toUpperCase(), - "exchange-listed": symbolInfo.exchange.toUpperCase(), + "exchange-traded": symbolInfo.exchange, + "exchange-listed": symbolInfo.exchange, "timezone": "America/New_York", "minmov": 1, "minmov2": 0, @@ -432,7 +423,7 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { "type": symbolInfo.type, "supported_resolutions": ["D", "2D", "3D", "W", "3W", "M", "6M"], "pricescale": 100, - "ticker": (symbolInfo.exchange + ':' + symbolInfo.name).toUpperCase(), + "ticker": symbolInfo.name.toUpperCase() }; }; @@ -631,16 +622,16 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendConfig(response); } else if (action === "/symbols" && !!query["symbol"]) { - this._sendSymbolInfo(removeExchangeFromSymbol(query["symbol"]), response); + this._sendSymbolInfo(query["symbol"], response); } else if (action === "/search") { this._sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); } else if (action === "/history") { - this._sendSymbolHistory(removeExchangeFromSymbol(query["symbol"]), query["from"], query["to"], query["resolution"].toLowerCase(), response); + this._sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response); } else if (action === "/quotes") { - this._sendQuotes(removeExchangeFromSymbols(query["symbols"]), response); + this._sendQuotes(query["symbols"], response); } else if (action === "/marks") { this._sendMarks(response); @@ -652,7 +643,7 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendTimescaleMarks(response); } else if (action === "/news") { - this._sendNews(removeExchangeFromSymbol(query["symbol"]), response); + this._sendNews(query["symbol"], response); } else if (action === "/futuresmag") { this._sendFuturesmag(response); From 9becbebc0a0577c80d78a667eea1e627c72996c7 Mon Sep 17 00:00:00 2001 From: Evgeniy Zhukovskiy Date: Tue, 2 Apr 2019 09:48:33 +0300 Subject: [PATCH 41/59] switch to oilprice --- request-processor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request-processor.js b/request-processor.js index c7bff25..4336b7c 100644 --- a/request-processor.js +++ b/request-processor.js @@ -609,8 +609,8 @@ RequestProcessor.prototype._sendNews = function (symbol, response) { RequestProcessor.prototype._sendFuturesmag = function (response) { var options = { - host: "www.futuresmag.com", - path: "/rss/all" + host: "www.oilprice.com", + path: "/rss/main" }; proxyRequest(http, options, response); From 4a375766bac6865c266d690cfadb95910fce2591 Mon Sep 17 00:00:00 2001 From: Marina Moshnogorskaya Date: Wed, 5 Feb 2020 09:56:52 +0300 Subject: [PATCH 42/59] Update Readme with launch instructions --- README.md | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 22e14a9..f47615a 100644 --- a/README.md +++ b/README.md @@ -3,8 +3,16 @@ UDF-compatible Quandl/Yahoo datafeed This repository contains a sample implementation of server-side UDF-compatible data source. -Register for free at www.quandl.com to get your free API key. +Register for free at www.quandl.com to get free API key. -Set QUANDL_API_KEY environment variable to your Quandl key before starting the feed. +Use NodeJS to launch yahoo.js with your Quandl key: -Use NodeJS to launch yahoo.js \ No newline at end of file +```bash +QUANDL_API_KEY=YOUR_KEY nodejs yahoo.js +``` +And change source URL in index.html file of Charting Library: + +```javascript +datafeed: new Datafeeds.UDFCompatibleDatafeed("http://localhost:8888") +``` +Save file and restart Charting Library server. From c431ff44cb9bf2a78668e7655e4efdc4edea055a Mon Sep 17 00:00:00 2001 From: Marina Moshnogorskaya Date: Thu, 6 Feb 2020 19:21:25 +0300 Subject: [PATCH 43/59] Update README.md --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index f47615a..4eafa7e 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,18 @@ -UDF-compatible Quandl/Yahoo datafeed +UDF-compatible Quandl/Yahoo data server ============== -This repository contains a sample implementation of server-side UDF-compatible data source. +This repository contains a sample of UDF-compatible data server. -Register for free at www.quandl.com to get free API key. +Register for free at www.quandl.com to get a free API key. -Use NodeJS to launch yahoo.js with your Quandl key: +Use NodeJS to launch `yahoo.js` with your Quandl key: ```bash QUANDL_API_KEY=YOUR_KEY nodejs yahoo.js ``` -And change source URL in index.html file of Charting Library: +Change the source URL in `index.html` file of the Charting Library: ```javascript datafeed: new Datafeeds.UDFCompatibleDatafeed("http://localhost:8888") ``` -Save file and restart Charting Library server. +Save the file and restart the Charting Library server. From 972a5962f4d9bd0b621661e51901f5a010d60726 Mon Sep 17 00:00:00 2001 From: Marina Moshnogorskaya Date: Mon, 27 Apr 2020 14:49:47 +0300 Subject: [PATCH 44/59] Move bar and timescale marks to last bar --- request-processor.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/request-processor.js b/request-processor.js index 4336b7c..70ac890 100644 --- a/request-processor.js +++ b/request-processor.js @@ -314,7 +314,7 @@ RequestProcessor.prototype._sendConfig = function (response) { RequestProcessor.prototype._sendMarks = function (response) { - var now = new Date(); + var now = new Date(this.lastBarTime) || new Date(); now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; var day = 60 * 60 * 24; @@ -341,7 +341,7 @@ RequestProcessor.prototype._sendTime = function (response) { }; RequestProcessor.prototype._sendTimescaleMarks = function (response) { - var now = new Date(); + var now = new Date(this.lastBarTime) || new Date(); now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; var day = 60 * 60 * 24; @@ -468,6 +468,7 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes if (quandlCache[key]) { var dataFromCache = filterDataPeriod(quandlCache[key], startDateTimestamp, endDateTimestamp); + this.lastBarTime = this.lastBarTime || +endDateTimestamp * 1000; logForData(dataFromCache, key, true); sendResult(JSON.stringify(dataFromCache)); return; From b0a3cdc1a1fc8f132f150461f91c01f68f3932f5 Mon Sep 17 00:00:00 2001 From: Marina Moshnogorskaya Date: Wed, 29 Apr 2020 12:21:36 +0300 Subject: [PATCH 45/59] Replace marks date calc with hardcoded timestamp --- request-processor.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/request-processor.js b/request-processor.js index 70ac890..ef53e97 100644 --- a/request-processor.js +++ b/request-processor.js @@ -314,7 +314,7 @@ RequestProcessor.prototype._sendConfig = function (response) { RequestProcessor.prototype._sendMarks = function (response) { - var now = new Date(this.lastBarTime) || new Date(); + var now = new Date(1522108800000); now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; var day = 60 * 60 * 24; @@ -322,7 +322,7 @@ RequestProcessor.prototype._sendMarks = function (response) { id: [0, 1, 2, 3, 4, 5], time: [now, now - day * 4, now - day * 7, now - day * 7, now - day * 15, now - day * 30], color: ["red", "blue", "green", "red", "blue", "green"], - text: ["Today", "4 days back", "7 days back + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", "7 days back once again", "15 days back", "30 days back"], + text: ["Red", "Blue", "Green + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", "Red again", "Blue", "Green"], label: ["A", "B", "CORE", "D", "EURO", "F"], labelFontColor: ["white", "white", "red", "#FFFFFF", "white", "#000"], minSize: [14, 28, 7, 40, 7, 14] @@ -341,7 +341,7 @@ RequestProcessor.prototype._sendTime = function (response) { }; RequestProcessor.prototype._sendTimescaleMarks = function (response) { - var now = new Date(this.lastBarTime) || new Date(); + var now = new Date(1522108800000); now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; var day = 60 * 60 * 24; @@ -468,7 +468,6 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes if (quandlCache[key]) { var dataFromCache = filterDataPeriod(quandlCache[key], startDateTimestamp, endDateTimestamp); - this.lastBarTime = this.lastBarTime || +endDateTimestamp * 1000; logForData(dataFromCache, key, true); sendResult(JSON.stringify(dataFromCache)); return; From a353552a174d4055fe34d4d8968ff1389cd726f5 Mon Sep 17 00:00:00 2001 From: Marina Moshnogorskaya Date: Thu, 7 May 2020 13:33:30 +0300 Subject: [PATCH 46/59] Rename and optimize marks date --- request-processor.js | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/request-processor.js b/request-processor.js index ef53e97..d154c79 100644 --- a/request-processor.js +++ b/request-processor.js @@ -314,13 +314,19 @@ RequestProcessor.prototype._sendConfig = function (response) { RequestProcessor.prototype._sendMarks = function (response) { - var now = new Date(1522108800000); - now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; + var lastMarkTimestamp = 1522108800; var day = 60 * 60 * 24; var marks = { id: [0, 1, 2, 3, 4, 5], - time: [now, now - day * 4, now - day * 7, now - day * 7, now - day * 15, now - day * 30], + time: [ + lastMarkTimestamp, + lastMarkTimestamp - day * 4, + lastMarkTimestamp - day * 7, + lastMarkTimestamp - day * 7, + lastMarkTimestamp - day * 15, + lastMarkTimestamp - day * 30 + ], color: ["red", "blue", "green", "red", "blue", "green"], text: ["Red", "Blue", "Green + Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua.", "Red again", "Blue", "Green"], label: ["A", "B", "CORE", "D", "EURO", "F"], @@ -341,42 +347,41 @@ RequestProcessor.prototype._sendTime = function (response) { }; RequestProcessor.prototype._sendTimescaleMarks = function (response) { - var now = new Date(1522108800000); - now = new Date(Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate())) / 1000; + var lastMarkTimestamp = 1522108800; var day = 60 * 60 * 24; var marks = [ { id: "tsm1", - time: now, + time: lastMarkTimestamp, color: "red", label: "A", tooltip: "" }, { id: "tsm2", - time: now - day * 4, + time: lastMarkTimestamp - day * 4, color: "blue", label: "D", - tooltip: ["Dividends: $0.56", "Date: " + new Date((now - day * 4) * 1000).toDateString()] + tooltip: ["Dividends: $0.56", "Date: " + new Date((lastMarkTimestamp - day * 4) * 1000).toDateString()] }, { id: "tsm3", - time: now - day * 7, + time: lastMarkTimestamp - day * 7, color: "green", label: "D", - tooltip: ["Dividends: $3.46", "Date: " + new Date((now - day * 7) * 1000).toDateString()] + tooltip: ["Dividends: $3.46", "Date: " + new Date((lastMarkTimestamp - day * 7) * 1000).toDateString()] }, { id: "tsm4", - time: now - day * 15, + time: lastMarkTimestamp - day * 15, color: "#999999", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"] }, { id: "tsm7", - time: now - day * 30, + time: lastMarkTimestamp - day * 30, color: "red", label: "E", tooltip: ["Earnings: $5.40", "Estimate: $5.00"] From ab19fbb7f5b5d73cc0f7530fa0135e57a6ee354b Mon Sep 17 00:00:00 2001 From: Evgeniy Timokhov Date: Thu, 21 Oct 2021 12:28:33 +0100 Subject: [PATCH 47/59] Fixed handling empty symbol name --- request-processor.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/request-processor.js b/request-processor.js index d154c79..ef03f9c 100644 --- a/request-processor.js +++ b/request-processor.js @@ -463,6 +463,14 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes console.log(dateForLogs() + "Return QUANDL result" + (isCached ? " from cache" : "") + ": " + key + ", from " + secondsToISO(fromCacheTime) + " to " + secondsToISO(toCacheTime)); } + symbol = (symbol || '').trim(); + + if (symbol.length === 0) { + console.log(dateForLogs() + "Invalid symbol=" + symbol); + sendError('Invalid symbol', response); + return; + } + console.log(dateForLogs() + "Got history request for " + symbol + ", " + resolution + " from " + secondsToISO(startDateTimestamp)+ " to " + secondsToISO(endDateTimestamp)); // always request all data to reduce number of requests to quandl From 016964bf4cd81303dc78c558b754a5a5014b4508 Mon Sep 17 00:00:00 2001 From: Evgeniy Timokhov Date: Mon, 1 Nov 2021 10:41:51 +0000 Subject: [PATCH 48/59] Update request-processor.js --- request-processor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request-processor.js b/request-processor.js index ef03f9c..dab4bf1 100644 --- a/request-processor.js +++ b/request-processor.js @@ -13,14 +13,14 @@ "use strict"; -var version = '2.0.4'; +var version = '2.1.0'; var https = require("https"); var http = require("http"); var quandlCache = {}; -var quandlCacheCleanupTime = 3 * 60 * 60 * 1000; // 3 hours +var quandlCacheCleanupTime = 24 * 60 * 60 * 1000; // 3 hours var quandlKeysValidateTime = 15 * 60 * 1000; // 15 minutes var quandlMinimumDate = '1970-01-01'; From 25e9bb873a9c5bb07d7640a7bd340039c7ff2e83 Mon Sep 17 00:00:00 2001 From: Evgeniy Timokhov Date: Mon, 1 Nov 2021 10:41:59 +0000 Subject: [PATCH 49/59] Update request-processor.js --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index dab4bf1..44cd1c0 100644 --- a/request-processor.js +++ b/request-processor.js @@ -20,7 +20,7 @@ var http = require("http"); var quandlCache = {}; -var quandlCacheCleanupTime = 24 * 60 * 60 * 1000; // 3 hours +var quandlCacheCleanupTime = 24 * 60 * 60 * 1000; // 24 hours var quandlKeysValidateTime = 15 * 60 * 1000; // 15 minutes var quandlMinimumDate = '1970-01-01'; From 125033b183e38c130a4ef21391b6f604176b9c1b Mon Sep 17 00:00:00 2001 From: Evgeniy Timokhov Date: Mon, 1 Nov 2021 10:43:19 +0000 Subject: [PATCH 50/59] Update request-processor.js --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index 44cd1c0..b86a6e8 100644 --- a/request-processor.js +++ b/request-processor.js @@ -641,7 +641,7 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); } else if (action === "/history") { - this._sendSymbolHistory(query["symbol"], query["from"], query["to"], query["resolution"].toLowerCase(), response); + this._sendSymbolHistory(query["symbol"], query["from"], query["to"], (query["resolution"] || "").toLowerCase(), response); } else if (action === "/quotes") { this._sendQuotes(query["symbols"], response); From 220a5386f98382e78f7a7a3959ecad2ab8e330c9 Mon Sep 17 00:00:00 2001 From: Edward Dewhurst Date: Tue, 31 May 2022 18:00:41 +0100 Subject: [PATCH 51/59] Add timescale mark earning shapes --- request-processor.js | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/request-processor.js b/request-processor.js index b86a6e8..44b5fd1 100644 --- a/request-processor.js +++ b/request-processor.js @@ -375,16 +375,25 @@ RequestProcessor.prototype._sendTimescaleMarks = function (response) { { id: "tsm4", time: lastMarkTimestamp - day * 15, - color: "#999999", + color: "red", label: "E", - tooltip: ["Earnings: $3.44", "Estimate: $3.60"] + tooltip: ["Earnings: $3.44", "Estimate: $3.60"], + shape: 'earningDown', }, { id: "tsm7", time: lastMarkTimestamp - day * 30, - color: "red", + color: "green", label: "E", - tooltip: ["Earnings: $5.40", "Estimate: $5.00"] + tooltip: ["Earnings: $5.40", "Estimate: $5.00"], + shape: 'earningUp', + }, + { + id: "tsm8", + time: now - day * 30, + color: "orange", + label: "S", + tooltip: ["Split: 4/1", "Date: " + new Date((now - day * 30) * 1000).toDateString()], }, ]; From 0a98122334bb33d94476a99205386cc089de7b19 Mon Sep 17 00:00:00 2001 From: Edward Dewhurst Date: Thu, 9 Jun 2022 15:53:27 +0100 Subject: [PATCH 52/59] Fix undefined variable error --- request-processor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/request-processor.js b/request-processor.js index 44b5fd1..c75c1f7 100644 --- a/request-processor.js +++ b/request-processor.js @@ -390,10 +390,10 @@ RequestProcessor.prototype._sendTimescaleMarks = function (response) { }, { id: "tsm8", - time: now - day * 30, + time: lastMarkTimestamp - day * 30, color: "orange", label: "S", - tooltip: ["Split: 4/1", "Date: " + new Date((now - day * 30) * 1000).toDateString()], + tooltip: ["Split: 4/1", "Date: " + new Date((lastMarkTimestamp - day * 30) * 1000).toDateString()], }, ]; From 0ba44e0a6395584eb82150981005db77a7106d85 Mon Sep 17 00:00:00 2001 From: ochernenko Date: Fri, 18 Nov 2022 18:42:37 +0400 Subject: [PATCH 53/59] CL-1388: Update timescale marks colors --- request-processor.js | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/request-processor.js b/request-processor.js index c75c1f7..a87d066 100644 --- a/request-processor.js +++ b/request-processor.js @@ -354,28 +354,28 @@ RequestProcessor.prototype._sendTimescaleMarks = function (response) { { id: "tsm1", time: lastMarkTimestamp, - color: "red", + color: "#F23645", label: "A", tooltip: "" }, { id: "tsm2", time: lastMarkTimestamp - day * 4, - color: "blue", + color: "#2962FF", label: "D", tooltip: ["Dividends: $0.56", "Date: " + new Date((lastMarkTimestamp - day * 4) * 1000).toDateString()] }, { id: "tsm3", time: lastMarkTimestamp - day * 7, - color: "green", + color: "#089981", label: "D", tooltip: ["Dividends: $3.46", "Date: " + new Date((lastMarkTimestamp - day * 7) * 1000).toDateString()] }, { id: "tsm4", time: lastMarkTimestamp - day * 15, - color: "red", + color: "#F23645", label: "E", tooltip: ["Earnings: $3.44", "Estimate: $3.60"], shape: 'earningDown', @@ -383,7 +383,7 @@ RequestProcessor.prototype._sendTimescaleMarks = function (response) { { id: "tsm7", time: lastMarkTimestamp - day * 30, - color: "green", + color: "#089981", label: "E", tooltip: ["Earnings: $5.40", "Estimate: $5.00"], shape: 'earningUp', @@ -391,7 +391,7 @@ RequestProcessor.prototype._sendTimescaleMarks = function (response) { { id: "tsm8", time: lastMarkTimestamp - day * 30, - color: "orange", + color: "#FF9800", label: "S", tooltip: ["Split: 4/1", "Date: " + new Date((lastMarkTimestamp - day * 30) * 1000).toDateString()], }, From 3784a056886dbee75ebe98761e4171e73eaecdf9 Mon Sep 17 00:00:00 2001 From: Mark Silverwood Date: Wed, 30 Nov 2022 16:07:01 +0000 Subject: [PATCH 54/59] add support for `countback` --- request-processor.js | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/request-processor.js b/request-processor.js index a87d066..8c4948f 100644 --- a/request-processor.js +++ b/request-processor.js @@ -205,12 +205,15 @@ function RequestProcessor(symbolsDatabase) { this._symbolsDatabase = symbolsDatabase; } -function filterDataPeriod(data, fromSeconds, toSeconds) { +function filterDataPeriod(data, fromSeconds, toSeconds, countback) { if (!data || !data.t) { return data; } - if (data.t[data.t.length - 1] < fromSeconds) { + var countbackInt = parseInt(countback, 10); + var countbackValid = !Number.isNaN(countbackInt) && countbackInt > 0; + + if (data.t[data.t.length - 1] < fromSeconds && !countbackValid) { return { s: 'no_data', nextTime: data.t[data.t.length - 1] @@ -242,7 +245,18 @@ function filterDataPeriod(data, fromSeconds, toSeconds) { s = 'no_data'; } - toIndex = Math.min(fromIndex + 1000, toIndex); // do not send more than 1000 bars for server capacity reasons + if (countbackValid) { + fromIndex = Math.max(0, toIndex - countbackInt); + } + + /** + * ! Do not send more than 1000 bars for server capacity reasons ! + * + * We are limiting the number of data points returned by sending the latest portion (newest dates). + * The datafeed should be aware of this behavior, so it can request the earlier data again. + * (CL won't ask for the missing data again, so the datafeed API needs to handle bundling the multiple requests together) + */ + fromIndex = Math.max(toIndex - 1000, fromIndex); return { t: data.t.slice(fromIndex, toIndex), @@ -449,7 +463,7 @@ RequestProcessor.prototype._sendSymbolInfo = function (symbolName, response) { response.end(); }; -RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimestamp, endDateTimestamp, resolution, response) { +RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimestamp, endDateTimestamp, resolution, countback, response) { function sendResult(content) { var header = Object.assign({}, defaultResponseHeader); header["Content-Length"] = content.length; @@ -480,7 +494,7 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes return; } - console.log(dateForLogs() + "Got history request for " + symbol + ", " + resolution + " from " + secondsToISO(startDateTimestamp)+ " to " + secondsToISO(endDateTimestamp)); + console.log(dateForLogs() + "Got history request for " + symbol + ", " + resolution + " from " + secondsToISO(startDateTimestamp)+ " to " + secondsToISO(endDateTimestamp) + (countback ? " [countback: " + countback + "]": "")); // always request all data to reduce number of requests to quandl var from = quandlMinimumDate; @@ -489,7 +503,7 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes var key = symbol + "|" + from + "|" + to; if (quandlCache[key]) { - var dataFromCache = filterDataPeriod(quandlCache[key], startDateTimestamp, endDateTimestamp); + var dataFromCache = filterDataPeriod(quandlCache[key], startDateTimestamp, endDateTimestamp, countback); logForData(dataFromCache, key, true); sendResult(JSON.stringify(dataFromCache)); return; @@ -506,7 +520,7 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes var address = "/api/v3/datatables/WIKI/PRICES.json" + "?api_key=" + quandlKey + // you should create a free account on quandl.com to get this key "&ticker=" + symbol + - "&date.gte=" + from + + "&date.gte=" + from + // this is the min quandl data (so we will get the full history and reduce number of requests) "&date.lte=" + to; console.log(dateForLogs() + "Sending request to quandl " + key + ". url=" + address); @@ -546,7 +560,7 @@ RequestProcessor.prototype._sendSymbolHistory = function (symbol, startDateTimes console.log(dateForLogs() + "Parsing returned empty result."); } - var filteredData = filterDataPeriod(data, startDateTimestamp, endDateTimestamp); + var filteredData = filterDataPeriod(data, startDateTimestamp, endDateTimestamp, countback); logForData(filteredData, key, false); sendResult(JSON.stringify(filteredData)); }); @@ -650,7 +664,7 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { this._sendSymbolSearchResults(query["query"], query["type"], query["exchange"], query["limit"], response); } else if (action === "/history") { - this._sendSymbolHistory(query["symbol"], query["from"], query["to"], (query["resolution"] || "").toLowerCase(), response); + this._sendSymbolHistory(query["symbol"], query["from"], query["to"], (query["resolution"] || "").toLowerCase(), query["countback"], response); } else if (action === "/quotes") { this._sendQuotes(query["symbols"], response); From 6740c509ec2f6209b7bcb3cfcf3a762b523d0969 Mon Sep 17 00:00:00 2001 From: Mark Silverwood Date: Wed, 31 May 2023 15:13:58 +0100 Subject: [PATCH 55/59] Fix deprecated warning message `SymbolInfo validation: field has_no_volume is deprecated, use visible_plots_set instead` --- request-processor.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index 8c4948f..9dd915c 100644 --- a/request-processor.js +++ b/request-processor.js @@ -446,7 +446,7 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { "pointvalue": 1, "session": "0930-1630", "has_intraday": false, - "has_no_volume": symbolInfo.type !== "stock", + "visible_plots_set": symbolInfo.type !== "stock" ? 'ohlc' : 'ohlcv', "description": symbolInfo.description.length > 0 ? symbolInfo.description : symbolInfo.name, "type": symbolInfo.type, "supported_resolutions": ["D", "2D", "3D", "W", "3W", "M", "6M"], From 25cd36a08b63029c80daf411612f0280e159d4be Mon Sep 17 00:00:00 2001 From: Mark Silverwood Date: Wed, 31 May 2023 16:54:05 +0100 Subject: [PATCH 56/59] add symbol and exchange logos --- logos.js | 138 +++++++++++++++++++++++++++++++++++++++++++ request-processor.js | 11 +++- symbols_database.js | 15 ++++- 3 files changed, 161 insertions(+), 3 deletions(-) create mode 100644 logos.js diff --git a/logos.js b/logos.js new file mode 100644 index 0000000..9cbaf27 --- /dev/null +++ b/logos.js @@ -0,0 +1,138 @@ +const exchangeLogos = { + NasdaqNM: 'https://s3-symbol-logo.tradingview.com/country/US.svg', + NYSE: 'https://s3-symbol-logo.tradingview.com/country/US.svg', + NCM: 'https://s3-symbol-logo.tradingview.com/country/US.svg', +}; + +exports.getExchangeLogoUrl = function (exchangeName) { + return exchangeLogos[exchangeName]; +}; + +const baseUrl = 'https://s3-symbol-logo.tradingview.com/'; + +const symbolLogoNames = { + A: 'agilent-technologies.svg', + AA: 'alcoa.svg', + AAL: 'american-airlines-group.svg', + AAPL: 'apple.svg', + ABBV: 'abbvie.svg', + ABT: 'abbott.svg', + // ACHN: '.svg', + ACI: 'albertsons.svg', + ACN: 'accenture.svg', + ADBE: 'adobe.svg', + ADSK: 'autodesk.svg', + AEO: 'american-eagle-outfitters.svg', + AGNC: 'agnc-investment.svg', + AIG: 'american-international-group.svg', + AKAM: 'akamai.svg', + // ALXN: '.svg', + AMAT: 'applied-materials.svg', + AMD: 'advanced-micro-devices.svg', + AMGN: 'amgen.svg', + AMZN: 'amazon.svg', + ANF: 'abercrombie-and-fitch.svg', + // ANR: '.svg', + APA: 'apa-corporation.svg', + // APC: '.svg', + ARC: 'arc-document-solutions.svg', + // ARIA: '.svg', + // ARNA: '.svg', + ARR: 'armour-residential-reit-7-series-c-cumulative-redeemable-preferred-stock-liquidation-preference-2500-per-share.svg', + // AUXL: '.svg', + AVGO: 'broadcom.svg', + // AVNR: '.svg', + // AWAY: '.svg', + AXP: 'american-express.svg', + AZO: 'autozone.svg', + BA: 'boeing.svg', + BAC: 'bank-of-america.svg', + BAX: 'baxter.svg', + BBBY: 'bed-bath-and-beyond.svg', + // BBT: '.svg', + BBY: 'best-buy.svg', + BIDU: 'baidu.svg', + BIIB: 'biogen.svg', + BK: 'bank-of-new-york-mellon.svg', + BLK: 'blackrock.svg', + BMY: 'bristol-myers-squibb.svg', + BP: 'bp.svg', + // BRCD: '.svg', + // BRCM: '.svg', + BTU: 'peabody-energy.svg', + C: 'citigroup.svg', + CHK: 'chesapeake-energy.svg', + CNP: 'centerpoint-energy.svg', + CSCO: 'cisco.svg', + D: 'dominion-energy.svg', + DAL: 'delta-air-lines.svg', + // DBD: '.svg', + DD: 'dupont-de-nemours.svg', + DDD: '3-d-systems.svg', + DE: 'deere.svg', + DECK: 'deckers-outdoor.svg', + DEI: 'douglas-emmett.svg', + DHI: 'dr-horton.svg', + DIS: 'walt-disney.svg', + DLTR: 'dollar-tree.svg', + // DNDN: '.svg', + DO: 'diamond-offshore-drilling.svg', + DOV: 'dover.svg', + DOW: 'dow.svg', + DRI: 'darden.svg', + DV: 'doubleverify.svg', + DVN: 'devon-energy.svg', + EA: 'electronic-arts.svg', + EBAY: 'ebay.svg', + EBIX: 'ebix.svg', + // ECYT: '.svg', + ED: 'consolidated-edison.svg', + // EMC: '.svg', + // ENT: '.svg', + ESI: 'element-solutions.svg', + // ESRX: '.svg', + // ETFC: '.svg', + EXC: 'exelon.svg', + EXPE: 'expedia.svg', + F: 'ford.svg', + FCEL: 'fuelcell-energy.svg', + // GALE: '.svg', + GD: 'general-dynamics.svg', + GE: 'general-electric.svg', + // GTAT: '.svg', + HD: 'home-depot.svg', + IBM: 'international-bus-mach.svg', + INTC: 'intel.svg', + JPM: 'jpmorgan-chase.svg', + // KERX: '.svg', + KO: 'coca-cola.svg', + LLY: 'eli-lilly.svg', + LUV: 'southwest.svg', + MCD: 'mcdonalds.svg', + MNST: 'monster-beverage.svg', + MO: 'altria.svg', + MSFT: 'microsoft.svg', + NLY: 'annaly-capital-management.svg', + NUS: 'nu-skin-enterprises.svg', + OLED: 'universal-display.svg', + // PNRA: '.svg', + RAD: 'rite-aid.svg', + SAM: 'boston-beer-co.svg', + // SCTY: '.svg', + SD: 'sandridge-energy.svg', + STZ: 'constellation-brands.svg', + T: 'at-and-t.svg', + UA: 'under-armour.svg', + USB: 'us-bancorp.svg', + VZ: 'verizon.svg', + WDC: 'western-digital.svg', + WFC: 'wells-fargo.svg', + // WLT: '.svg', + XOM: 'exxon.svg', +}; + +exports.getSymbolLogos = function (name) { + const image = symbolLogoNames[name]; + if (!image) return undefined; + return [baseUrl + image]; +}; diff --git a/request-processor.js b/request-processor.js index 9dd915c..6c3ff51 100644 --- a/request-processor.js +++ b/request-processor.js @@ -18,6 +18,8 @@ var version = '2.1.0'; var https = require("https"); var http = require("http"); +var logos = require("./logos"); + var quandlCache = {}; var quandlCacheCleanupTime = 24 * 60 * 60 * 1000; // 24 hours @@ -436,7 +438,7 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { throw "unknown_symbol " + symbolName; } - return { + var result = { "name": symbolInfo.name, "exchange-traded": symbolInfo.exchange, "exchange-listed": symbolInfo.exchange, @@ -453,6 +455,13 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { "pricescale": 100, "ticker": symbolInfo.name.toUpperCase() }; + + var logoUrls = logos.getSymbolLogos(symbolInfo.name); + if (logoUrls) { + result.logo_urls = logoUrls; + } + + return result; }; RequestProcessor.prototype._sendSymbolInfo = function (symbolName, response) { diff --git a/symbols_database.js b/symbols_database.js index fad6bd1..45450d9 100644 --- a/symbols_database.js +++ b/symbols_database.js @@ -6,6 +6,8 @@ "use strict"; +var logos = require("./logos"); + /* global exports */ var symbols = [{"name":"A","description":"Agilent Technologies Inc.","exchange":"NYSE","type":"stock"}, @@ -129,13 +131,22 @@ var symbols = [{"name":"A","description":"Agilent Technologies Inc.","exchange": function searchResultFromDatabaseItem(item) { - return { + const result = { symbol: item.name, full_name: item.name, description: item.description, exchange: item.exchange, - type: item.type + type: item.type, }; + const logoUrls = logos.getSymbolLogos(item.name); + if (logoUrls) { + result.logo_urls = logoUrls; + } + const exchangeLogo = logos.getExchangeLogoUrl(item.exchange); + if (exchangeLogo) { + result.exchange_logo = exchangeLogo; + } + return result; } From 75ec564d7bd673dbe37c1229c45411c9fab8a974 Mon Sep 17 00:00:00 2001 From: Mark Silverwood Date: Tue, 4 Jul 2023 17:49:27 +0100 Subject: [PATCH 57/59] CL-1691: add exchange logo url to symbol info --- request-processor.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/request-processor.js b/request-processor.js index 6c3ff51..29ecc1d 100644 --- a/request-processor.js +++ b/request-processor.js @@ -460,6 +460,10 @@ RequestProcessor.prototype._prepareSymbolInfo = function (symbolName) { if (logoUrls) { result.logo_urls = logoUrls; } + var exchangeLogo = logos.getExchangeLogoUrl(symbolInfo.exchange); + if (exchangeLogo) { + result.exchange_logo = exchangeLogo; + } return result; }; From da4e5476430dbbec96a36b84cfe02516fea33563 Mon Sep 17 00:00:00 2001 From: Mark Silverwood Date: Thu, 3 Aug 2023 14:58:43 +0100 Subject: [PATCH 58/59] add api endpoint to get tv top news stories --- request-processor.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/request-processor.js b/request-processor.js index 29ecc1d..a1f0a4b 100644 --- a/request-processor.js +++ b/request-processor.js @@ -656,6 +656,10 @@ RequestProcessor.prototype._sendNews = function (symbol, response) { proxyRequest(https, options, response); }; +RequestProcessor.prototype._sendTVNews = function (response) { + proxyRequest(https, 'https://www.tradingview.com/key-events/feed/', response); +}; + RequestProcessor.prototype._sendFuturesmag = function (response) { var options = { host: "www.oilprice.com", @@ -694,6 +698,9 @@ RequestProcessor.prototype.processRequest = function (action, query, response) { else if (action === "/news") { this._sendNews(query["symbol"], response); } + else if (action === "/tv_news") { + this._sendTVNews(response); + } else if (action === "/futuresmag") { this._sendFuturesmag(response); } else { From f8fb3f1c8325a40d90bbd08d03eb50f8965f39e2 Mon Sep 17 00:00:00 2001 From: Mark Silverwood <3482679+SlicedSilver@users.noreply.github.com> Date: Tue, 15 Oct 2024 15:15:44 +0100 Subject: [PATCH 59/59] Add user agent to requests --- request-processor.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/request-processor.js b/request-processor.js index a1f0a4b..c2393c6 100644 --- a/request-processor.js +++ b/request-processor.js @@ -650,7 +650,10 @@ RequestProcessor.prototype._sendQuotes = function (tickersString, response) { RequestProcessor.prototype._sendNews = function (symbol, response) { var options = { host: "feeds.finance.yahoo.com", - path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US" + path: "/rss/2.0/headline?s=" + symbol + "®ion=US&lang=en-US", + headers: { + 'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36', + }, }; proxyRequest(https, options, response);