From a3be900ddca3f889533f31074963160a3e7f782a Mon Sep 17 00:00:00 2001 From: Eric Lindvall Date: Wed, 17 Apr 2013 00:12:21 -0700 Subject: [PATCH 1/7] Experimenting with providing data to reporters This is a basic attempt at figuring out what it would look like to give some intermediate mechanisms to make reporters simpler. --- lib/metriks/reporter/counter_derivative.rb | 86 +++++++++++ .../flattening_registry_enumerator.rb | 85 +++++++++++ lib/metriks/reporter/graphite.rb | 63 +------- lib/metriks/reporter/librato_metrics.rb | 144 ++---------------- 4 files changed, 191 insertions(+), 187 deletions(-) create mode 100644 lib/metriks/reporter/counter_derivative.rb create mode 100644 lib/metriks/reporter/flattening_registry_enumerator.rb diff --git a/lib/metriks/reporter/counter_derivative.rb b/lib/metriks/reporter/counter_derivative.rb new file mode 100644 index 0000000..d4fb3f4 --- /dev/null +++ b/lib/metriks/reporter/counter_derivative.rb @@ -0,0 +1,86 @@ +module Metriks + module Reporter + class CounterDerivative + def initialize(reporter) + @reporter = reporter + @last = Hash.new + end + + def each(&block) + @registry.each do |name, metric| + case metric + when Metriks::Meter + if @last["#{name}.count"] + count = metric.count + block.call("#{name}.count", count - @last["#{name}.count"]) + @last["#{name}.count"] = count + end + when Metriks::Counter + block.call("#{name}.count", metric.count) + when Metriks::Gauge + block.call("#{name}.value", metric.value) + when Metriks::UtilizationTimer + if @last["#{name}.count"] + count = metric.count + block.call("#{name}.count", count - @last["#{name}.count"]) + @last["#{name}.count"] = count + end + + block.call("#{name}.min", metric.min) + block.call("#{name}.max", metric.max) + block.call("#{name}.median", metric.median) + block.call("#{name}.stddev", metric.stddev) + + block.call("#{name}.one_minute_utilization", metric.one_minute_utilization) + block.call("#{name}.five_minute_utilization", metric.five_minute_utilization) + block.call("#{name}.fifteen_minute_utilization", metric.fifteen_minute_utilization) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median) + block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) + block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) + block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) + block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) + block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) + when Metriks::Timer + if @last["#{name}.count"] + count = metric.count + block.call("#{name}.count", count - @last["#{name}.count"]) + @last["#{name}.count"] = count + end + + block.call("#{name}.min", metric.min) + block.call("#{name}.max", metric.max) + block.call("#{name}.median", metric.median) + block.call("#{name}.stddev", metric.stddev) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median) + block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) + block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) + block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) + block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) + block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) + when Metriks::Histogram + block.call("#{name}.count", metric.count) + block.call("#{name}.min", metric.min) + block.call("#{name}.max", metric.max) + block.call("#{name}.median", metric.median) + block.call("#{name}.stddev", metric.stddev) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median) + block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) + block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) + block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) + block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) + block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/metriks/reporter/flattening_registry_enumerator.rb b/lib/metriks/reporter/flattening_registry_enumerator.rb new file mode 100644 index 0000000..cffefb3 --- /dev/null +++ b/lib/metriks/reporter/flattening_registry_enumerator.rb @@ -0,0 +1,85 @@ +module Metriks + module Reporter + class FlatteningRegistryEnumerator + def initialize(registry) + @registry = registry + end + + def each(&block) + @registry.each do |name, metric| + case metric + when Metriks::Meter + block.call("#{name}.count", metric.count) + block.call("#{name}.one_minute_rate", metric.one_minute_rate) + block.call("#{name}.five_minute_rate", metric.five_minute_rate) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) + block.call("#{name}.mean_rate", metric.mean_rate) + when Metriks::Counter + block.call("#{name}.count", metric.count) + when Metriks::Gauge + block.call("#{name}.value", metric.value) + when Metriks::UtilizationTimer + block.call("#{name}.count", metric.count) + block.call("#{name}.one_minute_rate", metric.one_minute_rate) + block.call("#{name}.five_minute_rate", metric.five_minute_rate) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) + block.call("#{name}.mean_rate", metric.mean_rate) + + block.call("#{name}.min", metric.min) + block.call("#{name}.max", metric.max) + block.call("#{name}.median", metric.median) + block.call("#{name}.stddev", metric.stddev) + + block.call("#{name}.one_minute_utilization", metric.one_minute_utilization) + block.call("#{name}.five_minute_utilization", metric.five_minute_utilization) + block.call("#{name}.fifteen_minute_utilization", metric.fifteen_minute_utilization) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median) + block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) + block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) + block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) + block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) + block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) + when Metriks::Timer + block.call("#{name}.count", metric.count) + block.call("#{name}.one_minute_rate", metric.one_minute_rate) + block.call("#{name}.five_minute_rate", metric.five_minute_rate) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) + block.call("#{name}.mean_rate", metric.mean_rate) + + block.call("#{name}.min", metric.min) + block.call("#{name}.max", metric.max) + block.call("#{name}.median", metric.median) + block.call("#{name}.stddev", metric.stddev) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median) + block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) + block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) + block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) + block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) + block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) + when Metriks::Histogram + block.call("#{name}.count", metric.count) + block.call("#{name}.min", metric.min) + block.call("#{name}.max", metric.max) + block.call("#{name}.median", metric.median) + block.call("#{name}.stddev", metric.stddev) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median) + block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) + block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) + block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) + block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) + block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) + end + end + end + end + end +end \ No newline at end of file diff --git a/lib/metriks/reporter/graphite.rb b/lib/metriks/reporter/graphite.rb index cc2a347..533a0ad 100644 --- a/lib/metriks/reporter/graphite.rb +++ b/lib/metriks/reporter/graphite.rb @@ -47,71 +47,20 @@ def restart end def write - @registry.each do |name, metric| - case metric - when Metriks::Meter - write_metric name, metric, [ - :count, :one_minute_rate, :five_minute_rate, - :fifteen_minute_rate, :mean_rate - ] - when Metriks::Counter - write_metric name, metric, [ - :count - ] - when Metriks::Gauge - write_metric name, metric, [ - :value - ] - when Metriks::UtilizationTimer - write_metric name, metric, [ - :count, :one_minute_rate, :five_minute_rate, - :fifteen_minute_rate, :mean_rate, - :min, :max, :mean, :stddev, - :one_minute_utilization, :five_minute_utilization, - :fifteen_minute_utilization, :mean_utilization, - ], [ - :median, :get_95th_percentile - ] - when Metriks::Timer - write_metric name, metric, [ - :count, :one_minute_rate, :five_minute_rate, - :fifteen_minute_rate, :mean_rate, - :min, :max, :mean, :stddev - ], [ - :median, :get_95th_percentile - ] - when Metriks::Histogram - write_metric name, metric, [ - :count, :min, :max, :mean, :stddev - ], [ - :median, :get_95th_percentile - ] - end + Metriks::Reporter::FlatteningRegistryEnumerator.new(@registry).each do |name, value| + write_metric(name, value) end end - def write_metric(base_name, metric, keys, snapshot_keys = []) + def write_metric(name, value) time = Time.now.to_i - base_name = base_name.to_s.gsub(/ +/, '_') if @prefix - base_name = "#{@prefix}.#{base_name}" - end - - keys.flatten.each do |key| - name = key.to_s.gsub(/^get_/, '') - value = metric.send(key) - socket.write("#{base_name}.#{name} #{value} #{time}\n") + name = "#{@prefix}.#{name}" end - unless snapshot_keys.empty? - snapshot = metric.snapshot - snapshot_keys.flatten.each do |key| - name = key.to_s.gsub(/^get_/, '') - value = snapshot.send(key) - socket.write("#{base_name}.#{name} #{value} #{time}\n") - end - end + name = name.to_s.gsub(/ +/, '_') + socket.write("#{name} #{value} #{time}\n") rescue Errno::EPIPE socket.close end diff --git a/lib/metriks/reporter/librato_metrics.rb b/lib/metriks/reporter/librato_metrics.rb index 528970f..f58e78c 100644 --- a/lib/metriks/reporter/librato_metrics.rb +++ b/lib/metriks/reporter/librato_metrics.rb @@ -1,5 +1,5 @@ require 'metriks/time_tracker' -require 'net/https' +require 'librato/metrics' module Metriks::Reporter class LibratoMetrics @@ -9,6 +9,9 @@ def initialize(email, token, options = {}) @email = email @token = token + @client = Librato::Metrics::Client.new + @client.authenticate(@email, @token) + @prefix = options[:prefix] @source = options[:source] @@ -44,142 +47,23 @@ def restart end def write - gauges = [] - @registry.each do |name, metric| - gauges << case metric - when Metriks::Meter - prepare_metric name, metric, [ - :count, :one_minute_rate, :five_minute_rate, - :fifteen_minute_rate, :mean_rate - ] - when Metriks::Counter - prepare_metric name, metric, [ - :count - ] - when Metriks::Gauge - prepare_metric name, metric, [ - :value - ] - when Metriks::UtilizationTimer - prepare_metric name, metric, [ - :count, :one_minute_rate, :five_minute_rate, - :fifteen_minute_rate, :mean_rate, - :min, :max, :mean, :stddev, - :one_minute_utilization, :five_minute_utilization, - :fifteen_minute_utilization, :mean_utilization, - ], [ - :median, :get_95th_percentile - ] - when Metriks::Timer - prepare_metric name, metric, [ - :count, :one_minute_rate, :five_minute_rate, - :fifteen_minute_rate, :mean_rate, - :min, :max, :mean, :stddev - ], [ - :median, :get_95th_percentile - ] - when Metriks::Histogram - prepare_metric name, metric, [ - :count, :min, :max, :mean, :stddev - ], [ - :median, :get_95th_percentile - ] - end - end - - gauges.flatten! - - unless gauges.empty? - submit(form_data(gauges.flatten)) - end - end - - def submit(data) - url = URI.parse('https://metrics-api.librato.com/v1/metrics') - req = Net::HTTP::Post.new(url.path) - req.basic_auth(@email, @token) - req.set_form_data(data) + time = @time_tracker.now_floored + queue = @client.new_queue - http = Net::HTTP.new(url.host, url.port) - http.verify_mode = OpenSSL::SSL::VERIFY_PEER - http.use_ssl = true - store = OpenSSL::X509::Store.new - store.set_default_paths - http.cert_store = store - - case res = http.start { |http| http.request(req) } - when Net::HTTPSuccess, Net::HTTPRedirection - # OK - else - res.error! - end - end - - def form_data(metrics) - data = {} - - gauges = metrics.select { |m| m[:type] == "gauge" } - - gauges.each_with_index do |gauge, idx| - gauge.each do |key, value| - if value - data["gauges[#{idx}][#{key}]"] = value.to_s - end + Metriks::Reporter::CounterDerivative.new(@registry).each do |name, value| + if prefix + name = "#{prefix}.#{name}" end - end - counters = metrics.select { |m| m[:type] == "counter" } - - counters.each_with_index do |counter, idx| - counter.each do |key, value| - if value - data["counters[#{idx}][#{key}]"] = value.to_s - end - end - end - - data - end - - def prepare_metric(base_name, metric, keys, snapshot_keys = []) - results = [] - time = @time_tracker.now_floored - - base_name = base_name.to_s.gsub(/ +/, '_') - if @prefix - base_name = "#{@prefix}.#{base_name}" - end - - keys.flatten.each do |key| - name = key.to_s.gsub(/^get_/, '') - value = metric.send(key) - - results << { - :type => name.to_s == "count" ? "counter" : "gauge", - :name => "#{base_name}.#{name}", - :source => @source, + queue.add name => { + :source => @source, + :value => value, :measure_time => time, - :value => value + :type => 'gauge' } end - unless snapshot_keys.empty? - snapshot = metric.snapshot - snapshot_keys.flatten.each do |key| - name = key.to_s.gsub(/^get_/, '') - value = snapshot.send(key) - - results << { - :type => name.to_s == "count" ? "counter" : "gauge", - :name => "#{base_name}.#{name}", - :source => @source, - :measure_time => time, - :value => value - } - end - end - - results + queue.submit end end end From f6c6de4b397a7accc5b1de5965f3fe4904f2be74 Mon Sep 17 00:00:00 2001 From: Eric Lindvall Date: Wed, 17 Apr 2013 00:32:30 -0700 Subject: [PATCH 2/7] Look at combining the counter tracking --- .../flattening_registry_enumerator.rb | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/lib/metriks/reporter/flattening_registry_enumerator.rb b/lib/metriks/reporter/flattening_registry_enumerator.rb index cffefb3..67cf59b 100644 --- a/lib/metriks/reporter/flattening_registry_enumerator.rb +++ b/lib/metriks/reporter/flattening_registry_enumerator.rb @@ -1,29 +1,43 @@ module Metriks module Reporter class FlatteningRegistryEnumerator - def initialize(registry) + def initialize(registry, options = {}) @registry = registry + if options[:counter_derivitive] + @last = Hash.new { |h,k| h[k] = 0 } + end end def each(&block) @registry.each do |name, metric| case metric when Metriks::Meter - block.call("#{name}.count", metric.count) - block.call("#{name}.one_minute_rate", metric.one_minute_rate) - block.call("#{name}.five_minute_rate", metric.five_minute_rate) - block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) - block.call("#{name}.mean_rate", metric.mean_rate) + if @last + count = metric.count + block.call("#{name}.count", count - @last["#{name}.count"]) + @last["#{name}.count"] = count + else + block.call("#{name}.count", metric.count) + block.call("#{name}.one_minute_rate", metric.one_minute_rate) + block.call("#{name}.five_minute_rate", metric.five_minute_rate) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) + block.call("#{name}.mean_rate", metric.mean_rate) + end + when Metriks::Counter block.call("#{name}.count", metric.count) when Metriks::Gauge block.call("#{name}.value", metric.value) when Metriks::UtilizationTimer - block.call("#{name}.count", metric.count) - block.call("#{name}.one_minute_rate", metric.one_minute_rate) - block.call("#{name}.five_minute_rate", metric.five_minute_rate) - block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) - block.call("#{name}.mean_rate", metric.mean_rate) + if @last + count = metric.count + block.call("#{name}.count", count - @last["#{name}.count"]) + @last["#{name}.count"] = count + else + block.call("#{name}.one_minute_rate", metric.one_minute_rate) + block.call("#{name}.five_minute_rate", metric.five_minute_rate) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) + end block.call("#{name}.min", metric.min) block.call("#{name}.max", metric.max) @@ -43,11 +57,17 @@ def each(&block) block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) when Metriks::Timer - block.call("#{name}.count", metric.count) - block.call("#{name}.one_minute_rate", metric.one_minute_rate) - block.call("#{name}.five_minute_rate", metric.five_minute_rate) - block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) - block.call("#{name}.mean_rate", metric.mean_rate) + if @last + count = metric.count + block.call("#{name}.count", count - @last["#{name}.count"]) + @last["#{name}.count"] = count + else + block.call("#{name}.count", metric.count) + block.call("#{name}.one_minute_rate", metric.one_minute_rate) + block.call("#{name}.five_minute_rate", metric.five_minute_rate) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) + block.call("#{name}.mean_rate", metric.mean_rate) + end block.call("#{name}.min", metric.min) block.call("#{name}.max", metric.max) From e729482c91363b1e4c7574d731cb19ba657cc239 Mon Sep 17 00:00:00 2001 From: Eric Lindvall Date: Wed, 17 Apr 2013 00:57:51 -0700 Subject: [PATCH 3/7] Spelling --- lib/metriks/reporter/flattening_registry_enumerator.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/metriks/reporter/flattening_registry_enumerator.rb b/lib/metriks/reporter/flattening_registry_enumerator.rb index 67cf59b..d6ded15 100644 --- a/lib/metriks/reporter/flattening_registry_enumerator.rb +++ b/lib/metriks/reporter/flattening_registry_enumerator.rb @@ -3,7 +3,7 @@ module Reporter class FlatteningRegistryEnumerator def initialize(registry, options = {}) @registry = registry - if options[:counter_derivitive] + if options[:counter_derivative] @last = Hash.new { |h,k| h[k] = 0 } end end From d82475371331c9ad74b54d9d7a23a1075d4164f3 Mon Sep 17 00:00:00 2001 From: Eric Lindvall Date: Thu, 4 Jul 2013 16:41:16 -0700 Subject: [PATCH 4/7] Implement changes based on discussions --- lib/metriks/reporter/counter_derivative.rb | 86 -------- .../flattening_registry_enumerator.rb | 190 ++++++++++-------- lib/metriks/reporter/graphite.rb | 1 - lib/metriks/reporter/librato_metrics.rb | 52 +++-- test/graphite_reporter_test.rb | 4 +- 5 files changed, 139 insertions(+), 194 deletions(-) delete mode 100644 lib/metriks/reporter/counter_derivative.rb diff --git a/lib/metriks/reporter/counter_derivative.rb b/lib/metriks/reporter/counter_derivative.rb deleted file mode 100644 index d4fb3f4..0000000 --- a/lib/metriks/reporter/counter_derivative.rb +++ /dev/null @@ -1,86 +0,0 @@ -module Metriks - module Reporter - class CounterDerivative - def initialize(reporter) - @reporter = reporter - @last = Hash.new - end - - def each(&block) - @registry.each do |name, metric| - case metric - when Metriks::Meter - if @last["#{name}.count"] - count = metric.count - block.call("#{name}.count", count - @last["#{name}.count"]) - @last["#{name}.count"] = count - end - when Metriks::Counter - block.call("#{name}.count", metric.count) - when Metriks::Gauge - block.call("#{name}.value", metric.value) - when Metriks::UtilizationTimer - if @last["#{name}.count"] - count = metric.count - block.call("#{name}.count", count - @last["#{name}.count"]) - @last["#{name}.count"] = count - end - - block.call("#{name}.min", metric.min) - block.call("#{name}.max", metric.max) - block.call("#{name}.median", metric.median) - block.call("#{name}.stddev", metric.stddev) - - block.call("#{name}.one_minute_utilization", metric.one_minute_utilization) - block.call("#{name}.five_minute_utilization", metric.five_minute_utilization) - block.call("#{name}.fifteen_minute_utilization", metric.fifteen_minute_utilization) - - snapshot = metric.snapshot - - block.call("#{name}.median", snapshot.median) - block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) - block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) - block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) - block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) - block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) - when Metriks::Timer - if @last["#{name}.count"] - count = metric.count - block.call("#{name}.count", count - @last["#{name}.count"]) - @last["#{name}.count"] = count - end - - block.call("#{name}.min", metric.min) - block.call("#{name}.max", metric.max) - block.call("#{name}.median", metric.median) - block.call("#{name}.stddev", metric.stddev) - - snapshot = metric.snapshot - - block.call("#{name}.median", snapshot.median) - block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) - block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) - block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) - block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) - block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) - when Metriks::Histogram - block.call("#{name}.count", metric.count) - block.call("#{name}.min", metric.min) - block.call("#{name}.max", metric.max) - block.call("#{name}.median", metric.median) - block.call("#{name}.stddev", metric.stddev) - - snapshot = metric.snapshot - - block.call("#{name}.median", snapshot.median) - block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) - block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) - block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) - block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) - block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) - end - end - end - end - end -end \ No newline at end of file diff --git a/lib/metriks/reporter/flattening_registry_enumerator.rb b/lib/metriks/reporter/flattening_registry_enumerator.rb index d6ded15..22f3cf1 100644 --- a/lib/metriks/reporter/flattening_registry_enumerator.rb +++ b/lib/metriks/reporter/flattening_registry_enumerator.rb @@ -1,103 +1,115 @@ module Metriks module Reporter class FlatteningRegistryEnumerator + include Enumerable + def initialize(registry, options = {}) @registry = registry - if options[:counter_derivative] + unless options[:current_rate] @last = Hash.new { |h,k| h[k] = 0 } end + + if options[:percentiles] + @percentiles = options[:percentiles] + else + @percentiles = [ 0.75, 0.95, 0.98, 0.99, 0.999 ] + end end - + def each(&block) @registry.each do |name, metric| - case metric - when Metriks::Meter - if @last - count = metric.count - block.call("#{name}.count", count - @last["#{name}.count"]) - @last["#{name}.count"] = count - else - block.call("#{name}.count", metric.count) - block.call("#{name}.one_minute_rate", metric.one_minute_rate) - block.call("#{name}.five_minute_rate", metric.five_minute_rate) - block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) - block.call("#{name}.mean_rate", metric.mean_rate) - end - - when Metriks::Counter - block.call("#{name}.count", metric.count) - when Metriks::Gauge - block.call("#{name}.value", metric.value) - when Metriks::UtilizationTimer - if @last - count = metric.count - block.call("#{name}.count", count - @last["#{name}.count"]) - @last["#{name}.count"] = count - else - block.call("#{name}.one_minute_rate", metric.one_minute_rate) - block.call("#{name}.five_minute_rate", metric.five_minute_rate) - block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) - end - - block.call("#{name}.min", metric.min) - block.call("#{name}.max", metric.max) - block.call("#{name}.median", metric.median) - block.call("#{name}.stddev", metric.stddev) - - block.call("#{name}.one_minute_utilization", metric.one_minute_utilization) - block.call("#{name}.five_minute_utilization", metric.five_minute_utilization) - block.call("#{name}.fifteen_minute_utilization", metric.fifteen_minute_utilization) - - snapshot = metric.snapshot - - block.call("#{name}.median", snapshot.median) - block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) - block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) - block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) - block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) - block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) - when Metriks::Timer - if @last - count = metric.count - block.call("#{name}.count", count - @last["#{name}.count"]) - @last["#{name}.count"] = count - else - block.call("#{name}.count", metric.count) - block.call("#{name}.one_minute_rate", metric.one_minute_rate) - block.call("#{name}.five_minute_rate", metric.five_minute_rate) - block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate) - block.call("#{name}.mean_rate", metric.mean_rate) - end - - block.call("#{name}.min", metric.min) - block.call("#{name}.max", metric.max) - block.call("#{name}.median", metric.median) - block.call("#{name}.stddev", metric.stddev) - - snapshot = metric.snapshot - - block.call("#{name}.median", snapshot.median) - block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) - block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) - block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) - block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) - block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) - when Metriks::Histogram - block.call("#{name}.count", metric.count) - block.call("#{name}.min", metric.min) - block.call("#{name}.max", metric.max) - block.call("#{name}.median", metric.median) - block.call("#{name}.stddev", metric.stddev) - - snapshot = metric.snapshot - - block.call("#{name}.median", snapshot.median) - block.call("#{name}.75th_percentile", snapshot.get_75th_percentile) - block.call("#{name}.95th_percentile", snapshot.get_95th_percentile) - block.call("#{name}.98th_percentile", snapshot.get_98th_percentile) - block.call("#{name}.99th_percentile", snapshot.get_99th_percentile) - block.call("#{name}.999th_percentile", snapshot.get_999th_percentile) + for_metric(name, metric, &block) + end + end + + def for_metric(name, metric, &block) + name = name.to_s.gsub(/ +/, '_') + + case metric + when Metriks::Meter + if @last + count = metric.count + block.call(name, count - @last[name], metric.class) + @last[name] = count + else + block.call("#{name}.count", metric.count, metric.class) + block.call("#{name}.one_minute_rate", metric.one_minute_rate, metric.class) + block.call("#{name}.five_minute_rate", metric.five_minute_rate, metric.class) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate, metric.class) + block.call("#{name}.mean_rate", metric.mean_rate, metric.class) end + + when Metriks::Counter + block.call("#{name}", metric.count, metric.class) + when Metriks::Gauge + block.call("#{name}", metric.value, metric.class) + when Metriks::UtilizationTimer + if @last + count = metric.count + block.call(name, count - @last[name], metric.class) + @last[name] = count + else + block.call("#{name}.one_minute_rate", metric.one_minute_rate, metric.class) + block.call("#{name}.five_minute_rate", metric.five_minute_rate, metric.class) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate, metric.class) + end + + block.call("#{name}.min", metric.min, metric.class) + block.call("#{name}.max", metric.max, metric.class) + block.call("#{name}.stddev", metric.stddev, metric.class) + + block.call("#{name}.one_minute_utilization", metric.one_minute_utilization, metric.class) + block.call("#{name}.five_minute_utilization", metric.five_minute_utilization, metric.class) + block.call("#{name}.fifteen_minute_utilization", metric.fifteen_minute_utilization, metric.class) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median, metric.class) + + for_percentiles(name, metric, snapshot, &block) + + when Metriks::Timer + if @last + count = metric.count + block.call(name, count - @last[name], metric.class) + @last["#{name}.count"] = count + else + block.call("#{name}.count", metric.count, metric.class) + block.call("#{name}.one_minute_rate", metric.one_minute_rate, metric.class) + block.call("#{name}.five_minute_rate", metric.five_minute_rate, metric.class) + block.call("#{name}.fifteen_minute_rate", metric.fifteen_minute_rate, metric.class) + block.call("#{name}.mean_rate", metric.mean_rate, metric.class) + end + + block.call("#{name}.min", metric.min, metric.class) + block.call("#{name}.max", metric.max, metric.class) + block.call("#{name}.stddev", metric.stddev, metric.class) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median, metric.class) + + for_percentiles(name, metric, snapshot, &block) + + when Metriks::Histogram + block.call("#{name}.count", metric.count, metric.class) + block.call("#{name}.min", metric.min, metric.class) + block.call("#{name}.max", metric.max, metric.class) + block.call("#{name}.stddev", metric.stddev, metric.class) + + snapshot = metric.snapshot + + block.call("#{name}.median", snapshot.median, metric.class) + + for_percentiles(name, metric, snapshot, &block) + end + end + + + def for_percentiles(name, metric, snapshot, &block) + @percentiles.each do |percentile| + percentile_name = (percentile * 100).to_f.to_s.gsub(/0+$/, '').gsub('.', '') + block.call("#{name}.#{percentile_name}th_percentile", snapshot.value(percentile), metric.class) end end end diff --git a/lib/metriks/reporter/graphite.rb b/lib/metriks/reporter/graphite.rb index 533a0ad..a97abdd 100644 --- a/lib/metriks/reporter/graphite.rb +++ b/lib/metriks/reporter/graphite.rb @@ -59,7 +59,6 @@ def write_metric(name, value) name = "#{@prefix}.#{name}" end - name = name.to_s.gsub(/ +/, '_') socket.write("#{name} #{value} #{time}\n") rescue Errno::EPIPE socket.close diff --git a/lib/metriks/reporter/librato_metrics.rb b/lib/metriks/reporter/librato_metrics.rb index f58e78c..a36fcc4 100644 --- a/lib/metriks/reporter/librato_metrics.rb +++ b/lib/metriks/reporter/librato_metrics.rb @@ -1,5 +1,6 @@ require 'metriks/time_tracker' -require 'librato/metrics' +require 'net/https' +require 'metriks/reporter/flattening_registry_enumerator' module Metriks::Reporter class LibratoMetrics @@ -9,20 +10,17 @@ def initialize(email, token, options = {}) @email = email @token = token - @client = Librato::Metrics::Client.new - @client.authenticate(@email, @token) - @prefix = options[:prefix] @source = options[:source] - @registry = options[:registry] || Metriks::Registry.default + @registry = options[:registry] || Metriks::Registry.default @time_tracker = Metriks::TimeTracker.new(options[:interval] || 60) - @on_error = options[:on_error] || proc { |ex| } + @on_error = options[:on_error] || proc { |ex| } end def start @thread ||= Thread.new do - loop do + while true @time_tracker.sleep Thread.new do @@ -46,24 +44,46 @@ def restart start end + def submit(data) + url = URI.parse('https://metrics-api.librato.com/v1/metrics') + req = Net::HTTP::Post.new(url.path) + req.basic_auth(@email, @token) + req.set_form_data(data) + + http = Net::HTTP.new(url.host, url.port) + http.verify_mode = OpenSSL::SSL::VERIFY_PEER + http.use_ssl = true + store = OpenSSL::X509::Store.new + store.set_default_paths + http.cert_store = store + + case res = http.start { |http| http.request(req) } + when Net::HTTPSuccess, Net::HTTPRedirection + # OK + else + res.error! + end + end + def write time = @time_tracker.now_floored - queue = @client.new_queue - Metriks::Reporter::CounterDerivative.new(@registry).each do |name, value| + enumerator = Metriks::Reporter::FlatteningRegistryEnumerator.new(@registry) + + data = {} + + enumerator.each_with_index do |name, value, klass, idx| if prefix name = "#{prefix}.#{name}" end - queue.add name => { - :source => @source, - :value => value, - :measure_time => time, - :type => 'gauge' - } + data["gauges[#{idx}][name]"] = name.to_s + data["gauges[#{idx}][source]"] = @source + data["gauges[#{idx}][measure_time]"] = time.to_i + data["gauges[#{idx}][value]"] = value end - queue.submit + submit(data) end end end diff --git a/test/graphite_reporter_test.rb b/test/graphite_reporter_test.rb index 51dc9c4..0c59c15 100644 --- a/test/graphite_reporter_test.rb +++ b/test/graphite_reporter_test.rb @@ -35,7 +35,7 @@ def test_write @reporter.write assert_match /timer.testing.median \d/, @stringio.string - assert_match /gauge.testing.value 123/, @stringio.string - assert_match /gauge.testing.block.value 456/, @stringio.string + assert_match /gauge.testing 123/, @stringio.string + assert_match /gauge.testing.block 456/, @stringio.string end end From a4afd3d8ea8915d4939d4fec0629253ac56146b9 Mon Sep 17 00:00:00 2001 From: Eric Lindvall Date: Thu, 4 Jul 2013 16:42:27 -0700 Subject: [PATCH 5/7] Add require --- lib/metriks/reporter/graphite.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/metriks/reporter/graphite.rb b/lib/metriks/reporter/graphite.rb index a97abdd..3acae1a 100644 --- a/lib/metriks/reporter/graphite.rb +++ b/lib/metriks/reporter/graphite.rb @@ -1,4 +1,5 @@ require 'socket' +require 'metriks/reporter/flattening_registry_enumerator' module Metriks::Reporter class Graphite From 0b6e3379e6788709f22736da5e6978633ec61c44 Mon Sep 17 00:00:00 2001 From: Eric Lindvall Date: Mon, 15 Jul 2013 14:17:02 -0700 Subject: [PATCH 6/7] Ensure everything gets submitted properly --- lib/metriks/reporter/librato_metrics.rb | 2 +- test/librato_metrics_reporter_test.rb | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/metriks/reporter/librato_metrics.rb b/lib/metriks/reporter/librato_metrics.rb index a36fcc4..5f66b98 100644 --- a/lib/metriks/reporter/librato_metrics.rb +++ b/lib/metriks/reporter/librato_metrics.rb @@ -72,7 +72,7 @@ def write data = {} - enumerator.each_with_index do |name, value, klass, idx| + enumerator.each_with_index do |(name, value, klass), idx| if prefix name = "#{prefix}.#{name}" end diff --git a/test/librato_metrics_reporter_test.rb b/test/librato_metrics_reporter_test.rb index 4c98455..874de84 100644 --- a/test/librato_metrics_reporter_test.rb +++ b/test/librato_metrics_reporter_test.rb @@ -28,7 +28,10 @@ def test_write @registry.utilization_timer('utilization_timer.testing').update(1.5) @registry.gauge('gauge.testing') { 123 } - @reporter.expects(:submit) + @reporter.expects(:submit).with do |data| + data.detect { |(k,v)| k =~ /gauges\[\d+\]\[name\]/ && v == 'gauge.testing' } && + data.detect { |(k,v)| k =~ /gauges\[\d+\]\[value\]/ && v.to_s == '123' } + end @reporter.write end From 9994eba13f301e150037df5aed6ec34cbc499288 Mon Sep 17 00:00:00 2001 From: Andrew Date: Fri, 4 Oct 2013 18:08:12 -0700 Subject: [PATCH 7/7] Prevent 400 error from librato by submitting an empty body --- lib/metriks/reporter/librato_metrics.rb | 2 +- test/librato_metrics_reporter_test.rb | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/metriks/reporter/librato_metrics.rb b/lib/metriks/reporter/librato_metrics.rb index 5f66b98..9dc1bfc 100644 --- a/lib/metriks/reporter/librato_metrics.rb +++ b/lib/metriks/reporter/librato_metrics.rb @@ -83,7 +83,7 @@ def write data["gauges[#{idx}][value]"] = value end - submit(data) + submit(data) unless data.empty? end end end diff --git a/test/librato_metrics_reporter_test.rb b/test/librato_metrics_reporter_test.rb index 874de84..4d9ebad 100644 --- a/test/librato_metrics_reporter_test.rb +++ b/test/librato_metrics_reporter_test.rb @@ -35,4 +35,9 @@ def test_write @reporter.write end + + def test_empty_write + @reporter.expects(:submit).never + @reporter.write + end end