diff --git a/Gemfile b/Gemfile index 3965e2c..39d729d 100644 --- a/Gemfile +++ b/Gemfile @@ -24,3 +24,9 @@ group :encryption do gem 'rackup' gem 'sinatra' end + +group :opentelemetry do + gem 'opentelemetry-api' + gem 'opentelemetry-exporter-otlp' + gem 'opentelemetry-sdk' +end diff --git a/Gemfile.lock b/Gemfile.lock index d19776f..a9b491c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -62,6 +62,25 @@ GEM oj (3.16.10) bigdecimal (>= 3.0) ostruct (>= 0.2) + opentelemetry-api (1.7.0) + opentelemetry-common (0.22.0) + opentelemetry-api (~> 1.0) + opentelemetry-exporter-otlp (0.30.0) + google-protobuf (>= 3.18) + googleapis-common-protos-types (~> 1.3) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-sdk (~> 1.2) + opentelemetry-semantic_conventions + opentelemetry-registry (0.4.0) + opentelemetry-api (~> 1.1) + opentelemetry-sdk (1.9.0) + opentelemetry-api (~> 1.1) + opentelemetry-common (~> 0.20) + opentelemetry-registry (~> 0.2) + opentelemetry-semantic_conventions + opentelemetry-semantic_conventions (1.36.0) + opentelemetry-api (~> 1.0) ostruct (0.6.1) parallel (1.26.3) parser (3.3.6.0) @@ -134,6 +153,9 @@ PLATFORMS DEPENDENCIES minitest + opentelemetry-api + opentelemetry-exporter-otlp + opentelemetry-sdk puma rackup rake diff --git a/README.md b/README.md index 7c8376f..b51bd85 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ Prerequisites: * [encryption](encryption) - Demonstrates how to make a codec for end-to-end encryption. * [env_config](env_config) - Load client configuration from TOML files with programmatic overrides. * [message_passing_simple](message_passing_simple) - Simple workflow that accepts signals, queries, and updates. +* [open_telemetry](open_telemetry) - Demonstrates how to use OpenTelemetry tracing and metrics with the Ruby SDK * [patching](patching) - Demonstrates how to safely alter a workflow. * [polling/frequent](polling/frequent) - Implement a frequent polling mechanism inside an Activity. * [polling/infrequent](polling/infrequent) - Implement an infrequent polling mechanism using Temporal's automatic Activity Retry feature. diff --git a/open_telemetry/README.md b/open_telemetry/README.md new file mode 100644 index 0000000..154dda7 --- /dev/null +++ b/open_telemetry/README.md @@ -0,0 +1,33 @@ +# OpenTelemetry Sample + +Demonstrates how to use OpenTelemetry tracing and metrics with the Ruby SDK + +## How to Run + +First, in another terminal start up a Grafana OpenTelemetry instance which will +collect telemetry and provide the Grafana UI for viewing the data. + +```bash + docker compose up +``` + +In another terminal, start the worker + +```bash + bundle exec ruby worker.rb +``` + +Finally start the workflow + +```bash + bundle exec ruby starter.rb +``` + +You should be able to see the result in the terminal. + +To view the Grafana dashboard go to `http://localhost:3000` + +You can find the trace by clicking on the "Explore" tab, +selecting "Tempo" as the data source, and switching the query type to "Search". + +There will be a trace for `my-service` containing the workflow trace. diff --git a/open_telemetry/compose_greeting_activity.rb b/open_telemetry/compose_greeting_activity.rb new file mode 100644 index 0000000..d5fc5ae --- /dev/null +++ b/open_telemetry/compose_greeting_activity.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +require 'temporalio/activity' + +module OpenTelemetry + class ComposeGreetingActivity < Temporalio::Activity::Definition + def initialize(tracer) + @tracer = tracer + end + + def execute(name) + # Capture start time for histogram metric later + start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) + + # Run activity in our own span. Most users will not need to create their own spans in activities, they will just + # rely on the default spans implicitly created. This is just a sample to show it can be done. + @tracer.in_span('my-activity-span', attributes: { 'my-group-attr' => 'simple-activities' }) do + # Sleep for a second, then return + sleep(1) + "Hello, #{name}!" + ensure + # Custom metrics can be created inside activities + Temporalio::Activity::Context.current.metric_meter + .create_metric(:histogram, 'my-activity-histogram', value_type: :duration) + .with_additional_attributes({ 'my-group-attr' => 'simple-activities' }) + .record(Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time) + end + end + end +end diff --git a/open_telemetry/docker-compose.yaml b/open_telemetry/docker-compose.yaml new file mode 100644 index 0000000..2843433 --- /dev/null +++ b/open_telemetry/docker-compose.yaml @@ -0,0 +1,8 @@ +services: + grafana-dashboard: + image: grafana/otel-lgtm:latest + tty: true + ports: + - 3000:3000 + - 4317:4317 + - 4318:4318 diff --git a/open_telemetry/greeting_workflow.rb b/open_telemetry/greeting_workflow.rb new file mode 100644 index 0000000..4a8b9c5 --- /dev/null +++ b/open_telemetry/greeting_workflow.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'temporalio/workflow' +require 'temporalio/contrib/open_telemetry' +require_relative 'compose_greeting_activity' + +module OpenTelemetry + class GreetingWorkflow < Temporalio::Workflow::Definition + def initialize + # Custom metrics can be created inside workflows. Most users will not need to create custom metrics inside + # workflows, this just shows that it can be done. + @my_workflow_counter = Temporalio::Workflow.metric_meter.create_metric(:counter, 'my-workflow-counter') + .with_additional_attributes({ 'my-group-attr' => 'simple-workflows' }) + end + + def execute(name) + # Increment our custom metric + @my_workflow_counter.record(35) + + # We can create a span in the workflow too. This is just an example to show this can be done, most users will not + # create spans in workflows but rather rely on the defaults. + # + # This span is completed as soon as created because OpenTelemetry doesn't support spans that may have to be + # completed on different machines. The span will be parented to the outer workflow span. Whether the outer span is + # the "StartWorkflow" from the client or the "RunWorkflow" where it first ran depends on if this is replayed + # separately from where it started. See the Ruby SDK README for more details. + Temporalio::Contrib::OpenTelemetry::Workflow.with_completed_span( + 'my-workflow-span', + attributes: { 'my-group-attr' => 'simple-workflows' } + ) do + # The span will be the parent of the span created here to start the activity + Temporalio::Workflow.execute_activity( + ComposeGreetingActivity, + name, # Activity argument + start_to_close_timeout: 5 * 60 # 5 minutes + ) + end + end + end +end diff --git a/open_telemetry/starter.rb b/open_telemetry/starter.rb new file mode 100644 index 0000000..bd3e061 --- /dev/null +++ b/open_telemetry/starter.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +require 'opentelemetry/sdk' +require 'temporalio/client' +require 'temporalio/contrib/open_telemetry' +require 'temporalio/runtime' +require_relative 'greeting_workflow' +require_relative 'util' + +# Configure metrics and tracing +OpenTelemetry::Util.configure_metrics_and_tracing + +# Demonstrate that we can create a custom metric right on the runtime, though most users won't need this +Temporalio::Runtime.default.metric_meter.create_metric(:gauge, 'my-starter-gauge', value_type: :float) + .with_additional_attributes({ 'my-group-attr' => 'simple-starters' }) + .record(1.23) + +# Create a client with the tracing interceptor set using the tracer +tracer = OpenTelemetry.tracer_provider.tracer('temporal_ruby_sample', '0.1.0') +client = Temporalio::Client.connect( + 'localhost:7233', + 'default', + interceptors: [Temporalio::Contrib::OpenTelemetry::TracingInterceptor.new(tracer)] +) + +# Demonstrate an arbitrary outer span. Most users may not explicitly create outer spans before using clients and rather +# solely rely on the implicit ones created in the client via interceptor, but this demonstrates that it can be done. +tracer.in_span('my-client-span', attributes: { 'my-group-attr' => 'simple-client' }) do + # Run workflow + puts 'Executing workflow' + result = client.execute_workflow( + OpenTelemetry::GreetingWorkflow, + 'User', # Workflow argument + id: 'opentelemetry-sample-workflow-id', + task_queue: 'opentelemetry-sample' + ) + puts "Workflow result: #{result}" +end diff --git a/open_telemetry/util.rb b/open_telemetry/util.rb new file mode 100644 index 0000000..1f2ce16 --- /dev/null +++ b/open_telemetry/util.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'opentelemetry/exporter/otlp' +require 'opentelemetry/sdk' +require 'temporalio/contrib/open_telemetry' +require 'temporalio/runtime' + +module OpenTelemetry + module Util + def self.configure_metrics_and_tracing + # Before doing anything, configure the default runtime with OpenTelemetry metrics. Unlike OpenTelemetry tracing in + # Temporal, OpenTelemetry metrics does not use the Ruby OpenTelemetry library, but rather an internal one. + Temporalio::Runtime.default = Temporalio::Runtime.new( + telemetry: Temporalio::Runtime::TelemetryOptions.new( + metrics: Temporalio::Runtime::MetricsOptions.new( + opentelemetry: Temporalio::Runtime::OpenTelemetryMetricsOptions.new( + url: 'http://127.0.0.1:4317', + durations_as_seconds: true + ) + ) + ) + ) + # Globally configure the Ruby OpenTelemetry library for tracing purposes. As of this writing, OpenTelemetry Ruby + # does not support OTLP over gRPC, so we use the HTTP endpoint instead. + OpenTelemetry::SDK.configure do |c| + c.service_name = 'my-service' + c.use_all + # Can use a SimpleSpanProcessor instead of a BatchSpanProcessor, but batch is better for production and moves + # the span exporting outside of the workflow instead of synchronously inside the workflow context. + processor = OpenTelemetry::SDK::Trace::Export::BatchSpanProcessor.new( + OpenTelemetry::Exporter::OTLP::Exporter.new( + endpoint: 'http://localhost:4318/v1/traces' + ) + ) + c.add_span_processor(processor) + # We need to shutdown the batch span processor on process exit to flush spans + at_exit { processor.shutdown } + end + end + end +end diff --git a/open_telemetry/worker.rb b/open_telemetry/worker.rb new file mode 100644 index 0000000..d69abcf --- /dev/null +++ b/open_telemetry/worker.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'opentelemetry/sdk' +require 'temporalio/client' +require 'temporalio/contrib/open_telemetry' +require 'temporalio/runtime' +require 'temporalio/worker' +require_relative 'compose_greeting_activity' +require_relative 'greeting_workflow' +require_relative 'util' + +# Configure metrics and tracing +OpenTelemetry::Util.configure_metrics_and_tracing + +# Demonstrate that we can create a custom metric right on the runtime, though most users won't need this +Temporalio::Runtime.default.metric_meter.create_metric(:gauge, 'my-worker-gauge', value_type: :float) + .with_additional_attributes({ 'my-group-attr' => 'simple-workers' }) + .record(1.23) + +# Create a client with the tracing interceptor set using the tracer +tracer = OpenTelemetry.tracer_provider.tracer('opentelemetry_sample', '1.0.0') +client = Temporalio::Client.connect( + 'localhost:7233', + 'default', + interceptors: [Temporalio::Contrib::OpenTelemetry::TracingInterceptor.new(tracer)] +) + +# Run worker +worker = Temporalio::Worker.new( + client:, + task_queue: 'opentelemetry-sample', + activities: [OpenTelemetry::ComposeGreetingActivity.new(tracer)], + workflows: [OpenTelemetry::GreetingWorkflow] +) +puts 'Starting worker (ctrl+c to exit)' +worker.run(shutdown_signals: ['SIGINT']) diff --git a/test/open_telemetry/greeting_workflow_test.rb b/test/open_telemetry/greeting_workflow_test.rb new file mode 100644 index 0000000..e8d3193 --- /dev/null +++ b/test/open_telemetry/greeting_workflow_test.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +require 'test' +require 'open_telemetry/compose_greeting_activity' +require 'open_telemetry/greeting_workflow' +require 'opentelemetry/sdk' +require 'securerandom' +require 'temporalio/client' +require 'temporalio/contrib/open_telemetry' +require 'temporalio/testing' +require 'temporalio/worker' + +module OpenTelemetry + class GreetingWorkflowTest < Test + def test_workflow + # Setup in memory buffer for telemetry events + metrics_buffer = Temporalio::Runtime::MetricBuffer.new(1024) + runtime = Temporalio::Runtime.new( + telemetry: Temporalio::Runtime::TelemetryOptions.new( + metrics: Temporalio::Runtime::MetricsOptions.new( + buffer: metrics_buffer + ) + ) + ) + tracer = OpenTelemetry.tracer_provider.tracer('opentelemetry_sample_test', '1.0.0') + Temporalio::Testing::WorkflowEnvironment.start_local( + runtime:, + interceptors: [Temporalio::Contrib::OpenTelemetry::TracingInterceptor.new(tracer)] + ) do |env| + # Run workflow in a worker + env.client + worker = Temporalio::Worker.new( + client: env.client, + task_queue: "tq-#{SecureRandom.uuid}", + activities: [ComposeGreetingActivity.new(tracer)], + workflows: [GreetingWorkflow] + ) + result = worker.run do + handle = env.client.start_workflow( + GreetingWorkflow, 'Temporal', + id: "wf-#{SecureRandom.uuid}", + task_queue: worker.task_queue + ) + handle.result + end + assert_equal 'Hello, Temporal!', result + assert !metrics_buffer.retrieve_updates.empty? + end + end + end +end