From 8691a4ada14de39a607f96ea128184da40168b90 Mon Sep 17 00:00:00 2001 From: Frank Olbricht Date: Sat, 1 Jun 2024 10:57:06 +0200 Subject: [PATCH 1/7] [rubygems/rubygems] Use IMDSv2 for S3 instance credentials https://github.com/rubygems/rubygems/commit/fa1c51ef59 --- lib/rubygems/s3_uri_signer.rb | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/rubygems/s3_uri_signer.rb b/lib/rubygems/s3_uri_signer.rb index 0d8e9e82851654..bdd272a77a00b7 100644 --- a/lib/rubygems/s3_uri_signer.rb +++ b/lib/rubygems/s3_uri_signer.rb @@ -146,18 +146,20 @@ def ec2_metadata_credentials_json require_relative "request" require_relative "request/connection_pools" require "json" - - iam_info = ec2_metadata_request(EC2_IAM_INFO) + token = ec2_metadata_token + iam_info = ec2_metadata_request(EC2_IAM_INFO, token) # Expected format: arn:aws:iam:::instance-profile/ role_name = iam_info["InstanceProfileArn"].split("/").last - ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name) + ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name, token) end - def ec2_metadata_request(url) + def ec2_metadata_request(url, token) uri = Gem::URI(url) @request_pool ||= create_request_pool(uri) request = Gem::Request.new(uri, Gem::Net::HTTP::Get, nil, @request_pool) - response = request.fetch + response = request.fetch do |req| + req.add_field "X-aws-ec2-metadata-token", token + end case response when Gem::Net::HTTPOK then @@ -167,6 +169,22 @@ def ec2_metadata_request(url) end end + def ec2_metadata_token + uri = Gem::URI(EC2_IAM_TOKEN) + @request_pool ||= create_request_pool(uri) + request = Gem::Request.new(uri, Gem::Net::HTTP::Put, nil, @request_pool) + response = request.fetch do |req| + req.add_field "X-aws-ec2-metadata-token-ttl-seconds", 60 + end + + case response + when Gem::Net::HTTPOK then + response.body + else + raise InstanceProfileError.new("Unable to fetch AWS metadata from #{uri}: #{response.message} #{response.code}") + end + end + def create_request_pool(uri) proxy_uri = Gem::Request.proxy_uri(Gem::Request.get_proxy_from_env(uri.scheme)) certs = Gem::Request.get_cert_files @@ -174,6 +192,7 @@ def create_request_pool(uri) end BASE64_URI_TRANSLATE = { "+" => "%2B", "/" => "%2F", "=" => "%3D", "\n" => "" }.freeze + EC2_IAM_TOKEN = "http://169.254.169.254/latest/api/token" EC2_IAM_INFO = "http://169.254.169.254/latest/meta-data/iam/info" EC2_IAM_SECURITY_CREDENTIALS = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" end From 720ae3285e26ba09b173b9f9fe0fab47fd508ff5 Mon Sep 17 00:00:00 2001 From: pjsk Date: Tue, 27 May 2025 17:42:03 -0700 Subject: [PATCH 2/7] [rubygems/rubygems] make things a bit more testable https://github.com/rubygems/rubygems/commit/29c085f5f5 --- lib/rubygems/s3_uri_signer.rb | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/rubygems/s3_uri_signer.rb b/lib/rubygems/s3_uri_signer.rb index bdd272a77a00b7..41c25bc77ee869 100644 --- a/lib/rubygems/s3_uri_signer.rb +++ b/lib/rubygems/s3_uri_signer.rb @@ -147,6 +147,7 @@ def ec2_metadata_credentials_json require_relative "request/connection_pools" require "json" token = ec2_metadata_token + iam_info = ec2_metadata_request(EC2_IAM_INFO, token) # Expected format: arn:aws:iam:::instance-profile/ role_name = iam_info["InstanceProfileArn"].split("/").last @@ -154,9 +155,8 @@ def ec2_metadata_credentials_json end def ec2_metadata_request(url, token) - uri = Gem::URI(url) - @request_pool ||= create_request_pool(uri) - request = Gem::Request.new(uri, Gem::Net::HTTP::Get, nil, @request_pool) + request = ec2_iam_request(Gem::URI(url), Gem::Net::HTTP::Get) + response = request.fetch do |req| req.add_field "X-aws-ec2-metadata-token", token end @@ -170,9 +170,8 @@ def ec2_metadata_request(url, token) end def ec2_metadata_token - uri = Gem::URI(EC2_IAM_TOKEN) - @request_pool ||= create_request_pool(uri) - request = Gem::Request.new(uri, Gem::Net::HTTP::Put, nil, @request_pool) + request = ec2_iam_request(Gem::URI(EC2_IAM_TOKEN), Gem::Net::HTTP::Put) + response = request.fetch do |req| req.add_field "X-aws-ec2-metadata-token-ttl-seconds", 60 end @@ -185,6 +184,14 @@ def ec2_metadata_token end end + def ec2_iam_request(uri, verb) + @request_pool ||= {} + @request_pool[uri] ||= create_request_pool(uri) + pool = @request_pool[uri] + + Gem::Request.new(uri, verb, nil, pool) + end + def create_request_pool(uri) proxy_uri = Gem::Request.proxy_uri(Gem::Request.get_proxy_from_env(uri.scheme)) certs = Gem::Request.get_cert_files From 23b34517bd8f62e50e2e5f7f22039a713aa32cc8 Mon Sep 17 00:00:00 2001 From: pjsk Date: Tue, 27 May 2025 17:43:37 -0700 Subject: [PATCH 3/7] [rubygems/rubygems] Surgery on test code to make fallback to imdsv1 easier to test https://github.com/rubygems/rubygems/commit/5b4eece722 --- test/rubygems/test_gem_remote_fetcher_s3.rb | 266 ++++++++++++++++---- 1 file changed, 223 insertions(+), 43 deletions(-) diff --git a/test/rubygems/test_gem_remote_fetcher_s3.rb b/test/rubygems/test_gem_remote_fetcher_s3.rb index e3aaa7a691f0a4..6c6f847b092304 100644 --- a/test/rubygems/test_gem_remote_fetcher_s3.rb +++ b/test/rubygems/test_gem_remote_fetcher_s3.rb @@ -8,24 +8,91 @@ class TestGemRemoteFetcherS3 < Gem::TestCase include Gem::DefaultUserInteraction - def setup - super + class FakeGemRequest < Gem::Request - @a1, @a1_gem = util_gem "a", "1" do |s| - s.executables << "a_bin" + attr_reader :last_request, :uri + + # Override perform_request to stub things + def perform_request(request) + @last_request = request + @response end - @a1.loaded_from = File.join(@gemhome, "specifications", @a1.full_name) + def set_response(response) + @response = response + end end - def assert_fetch_s3(url, signature, token=nil, region="us-east-1", instance_profile_json=nil, method="GET") - fetcher = Gem::RemoteFetcher.new nil - @fetcher = fetcher - $fetched_uri = nil - $instance_profile = instance_profile_json + class FakeS3URISigner < Gem::S3URISigner + # Convenience method to output the recent aws iam queries made in tests + # this outputs the verb, path, and any non-generic headers + def recent_aws_query_logs + sreqs = @aws_iam_calls.map do |c| + r = c.last_request + s = +"#{r.method} #{c.uri}\n" + r.each_header do |key, v| + # Only include headers that start with x- + next unless key.start_with?("x-") + s << " #{key}=#{v}\n" + end + s + end + + sreqs.join("") + end - def fetcher.request(uri, request_class, last_modified = nil) - $fetched_uri = uri + def initialize(uri, method) + @aws_iam_calls = [] + super + end + + def ec2_iam_request(uri, verb) + fake_s3_request = FakeGemRequest.new(uri, verb, nil, nil) + @aws_iam_calls << fake_s3_request + + case uri.to_s + when "http://169.254.169.254/latest/api/token" + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body + "mysecrettoken" + end + fake_s3_request.set_response(res) + + when "http://169.254.169.254/latest/meta-data/iam/info" + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body + <<~JSON + { + "Code": "Success", + "LastUpdated": "2023-05-27:05:05", + "InstanceProfileArn": "arn:aws:iam::somesecretid:instance-profile/TestRole", + "InstanceProfileId": "SOMEPROFILEID" + } + JSON + end + fake_s3_request.set_response(res) + + when "http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole" + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body + $instance_profile + end + fake_s3_request.set_response(res) + + else + raise "Unexpected request to #{uri}" + end + + fake_s3_request + end + end + + class FakeGemFetcher < Gem::RemoteFetcher + + attr_reader :fetched_uri, :last_s3_uri_signer + + def request(uri, request_class, last_modified = nil) + @fetched_uri = uri res = Gem::Net::HTTPOK.new nil, 200, nil def res.body "success" @@ -33,28 +100,71 @@ def res.body res end - def fetcher.s3_uri_signer(uri, method) - require "json" - s3_uri_signer = Gem::S3URISigner.new(uri, method) - def s3_uri_signer.ec2_metadata_credentials_json - JSON.parse($instance_profile) - end - # Running sign operation to make sure uri.query is not mutated - s3_uri_signer.sign - raise "URI query is not empty: #{uri.query}" unless uri.query.nil? - s3_uri_signer + def s3_uri_signer(uri, method) + @last_s3_uri_signer = FakeS3URISigner.new(uri, method) + end + end + + def setup + super + + @a1, @a1_gem = util_gem "a", "1" do |s| + s.executables << "a_bin" end + @a1.loaded_from = File.join(@gemhome, "specifications", @a1.full_name) + end + + def assert_fetched_s3_with_imds_v2 + # Three API requests: + # 1. Get the token + # 2. Lookup profile details + # 3. Query the credentials + expected = <<~TEXT + PUT http://169.254.169.254/latest/api/token + x-aws-ec2-metadata-token-ttl-seconds=60 + GET http://169.254.169.254/latest/meta-data/iam/info + x-aws-ec2-metadata-token=mysecrettoken + GET http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole + x-aws-ec2-metadata-token=mysecrettoken + TEXT + recent_aws_query_logs = @fetcher.last_s3_uri_signer.recent_aws_query_logs + assert_equal(expected.strip, recent_aws_query_logs.strip) + end + + def assert_fetch_s3(url:, signature:, token: nil, region: "us-east-1", instance_profile_json: nil, method: "GET") + @fetcher = FakeGemFetcher.new nil + $instance_profile = instance_profile_json res = fetcher.fetch_s3 Gem::URI.parse(url), nil, (method == "HEAD") - assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T051941Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", $fetched_uri.to_s + assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T051941Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", @fetcher.fetched_uri.to_s if method == "HEAD" assert_equal 200, res.code else assert_equal "success", res end + + # Validation for EC2 IAM signing + if Gem.configuration[:s3_source]&.dig("my-bucket", :provider) == "instance_profile" + # Three API requests: + # 1. Get the token + # 2. Lookup profile details + # 3. Query the credentials + expected = <<~TEXT + PUT http://169.254.169.254/latest/api/token + x-aws-ec2-metadata-token-ttl-seconds=60 + GET http://169.254.169.254/latest/meta-data/iam/info + x-aws-ec2-metadata-token=mysecrettoken + GET http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole + x-aws-ec2-metadata-token=mysecrettoken + TEXT + recent_aws_query_logs = @fetcher.last_s3_uri_signer.recent_aws_query_logs + assert_equal(expected.strip, recent_aws_query_logs.strip) + else + assert_equal("", @fetcher.last_s3_uri_signer.recent_aws_query_logs) + end ensure - $fetched_uri = nil + $instance_profile = nil end def test_fetch_s3_config_creds @@ -63,7 +173,10 @@ def test_fetch_s3_config_creds } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", + ) end ensure Gem.configuration[:s3_source] = nil @@ -79,7 +192,15 @@ def test_fetch_s3_head_request region = "us-east-1" instance_profile_json = nil method = "HEAD" - assert_fetch_s3 url, "a3c6cf9a2db62e85f4e57f8fc8ac8b5ff5c1fdd4aeef55935d05e05174d9c885", token, region, instance_profile_json, method + + assert_fetch_s3( + url: url, + signature: "a3c6cf9a2db62e85f4e57f8fc8ac8b5ff5c1fdd4aeef55935d05e05174d9c885", + token: token, + region: region, + instance_profile_json: instance_profile_json, + method: method + ) end ensure Gem.configuration[:s3_source] = nil @@ -91,7 +212,11 @@ def test_fetch_s3_config_creds_with_region } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", nil, "us-west-2" + assert_fetch_s3( + url: url, + signature: "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", + region: "us-west-2" + ) end ensure Gem.configuration[:s3_source] = nil @@ -103,7 +228,11 @@ def test_fetch_s3_config_creds_with_token } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", "testtoken" + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken" + ) end ensure Gem.configuration[:s3_source] = nil @@ -118,7 +247,10 @@ def test_fetch_s3_env_creds } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + ) end ensure ENV.each_key {|key| ENV.delete(key) if key.start_with?("AWS") } @@ -134,7 +266,12 @@ def test_fetch_s3_env_creds_with_region } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", nil, "us-west-2" + assert_fetch_s3( + url: url, + signature: "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", + token: nil, + region: "us-west-2" + ) end ensure ENV.each_key {|key| ENV.delete(key) if key.start_with?("AWS") } @@ -150,7 +287,11 @@ def test_fetch_s3_env_creds_with_token } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", "testtoken" + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken" + ) end ensure ENV.each_key {|key| ENV.delete(key) if key.start_with?("AWS") } @@ -160,7 +301,10 @@ def test_fetch_s3_env_creds_with_token def test_fetch_s3_url_creds url = "s3://testuser:testpass@my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + ) end end @@ -171,8 +315,13 @@ def test_fetch_s3_instance_profile_creds url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", nil, "us-east-1", - '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", + region: "us-east-1", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + ) + assert_fetched_s3_with_imds_v2 end ensure Gem.configuration[:s3_source] = nil @@ -185,8 +334,13 @@ def test_fetch_s3_instance_profile_creds_with_region url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", nil, "us-west-2", - '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + assert_fetch_s3( + url: url, + signature: "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", + region: "us-west-2", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + ) + assert_fetched_s3_with_imds_v2 end ensure Gem.configuration[:s3_source] = nil @@ -199,14 +353,40 @@ def test_fetch_s3_instance_profile_creds_with_token url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", "testtoken", "us-east-1", - '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken", + region: "us-east-1", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' + ) + assert_fetched_s3_with_imds_v2 + end + ensure + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_instance_profile_creds_with_fallback + Gem.configuration[:s3_source] = { + "my-bucket" => { provider: "instance_profile" }, + } + + url = "s3://my-bucket/gems/specs.4.8.gz" + Time.stub :now, Time.at(1_561_353_581) do + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken", + region: "us-east-1", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' + ) + assert_fetched_s3_with_imds_v2 end ensure Gem.configuration[:s3_source] = nil end - def refute_fetch_s3(url, expected_message) + def refute_fetch_s3(url:, expected_message:) fetcher = Gem::RemoteFetcher.new nil @fetcher = fetcher @@ -219,7 +399,7 @@ def refute_fetch_s3(url, expected_message) def test_fetch_s3_no_source_key url = "s3://my-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "no s3_source key exists in .gemrc" + refute_fetch_s3(url: url, expected_message: "no s3_source key exists in .gemrc") end def test_fetch_s3_no_host @@ -228,7 +408,7 @@ def test_fetch_s3_no_host } url = "s3://other-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "no key for host other-bucket in s3_source in .gemrc" + refute_fetch_s3(url: url, expected_message: "no key for host other-bucket in s3_source in .gemrc") ensure Gem.configuration[:s3_source] = nil end @@ -237,7 +417,7 @@ def test_fetch_s3_no_id Gem.configuration[:s3_source] = { "my-bucket" => { secret: "testpass" } } url = "s3://my-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "s3_source for my-bucket missing id or secret" + refute_fetch_s3(url: url, expected_message: "s3_source for my-bucket missing id or secret") ensure Gem.configuration[:s3_source] = nil end @@ -246,7 +426,7 @@ def test_fetch_s3_no_secret Gem.configuration[:s3_source] = { "my-bucket" => { id: "testuser" } } url = "s3://my-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "s3_source for my-bucket missing id or secret" + refute_fetch_s3(url: url, expected_message: "s3_source for my-bucket missing id or secret") ensure Gem.configuration[:s3_source] = nil end From 01ae9e4fb095db753291d65ae6d56411d17386a7 Mon Sep 17 00:00:00 2001 From: pjsk Date: Wed, 28 May 2025 16:56:54 -0700 Subject: [PATCH 4/7] [rubygems/rubygems] implement fallback https://github.com/rubygems/rubygems/commit/e09a6ec815 --- lib/rubygems/s3_uri_signer.rb | 39 ++++++++--- test/rubygems/test_gem_remote_fetcher_s3.rb | 72 +++++++++++++-------- 2 files changed, 75 insertions(+), 36 deletions(-) diff --git a/lib/rubygems/s3_uri_signer.rb b/lib/rubygems/s3_uri_signer.rb index 41c25bc77ee869..148cba38c47a9f 100644 --- a/lib/rubygems/s3_uri_signer.rb +++ b/lib/rubygems/s3_uri_signer.rb @@ -1,11 +1,14 @@ # frozen_string_literal: true require_relative "openssl" +require_relative "user_interaction" ## # S3URISigner implements AWS SigV4 for S3 Source to avoid a dependency on the aws-sdk-* gems # More on AWS SigV4: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html class Gem::S3URISigner + include Gem::UserInteraction + class ConfigurationError < Gem::Exception def initialize(message) super message @@ -146,19 +149,40 @@ def ec2_metadata_credentials_json require_relative "request" require_relative "request/connection_pools" require "json" + + # First try V2 fallback to V1 + res = nil + begin + res = ec2_metadata_credentials_imds_v2 + rescue InstanceProfileError + alert_warning "Unable to access ec2 credentials via IMDSv2, falling back to IMDSv1" + res = ec2_metadata_credentials_imds_v1 + end + res + end + + def ec2_metadata_credentials_imds_v2 token = ec2_metadata_token + iam_info = ec2_metadata_request(EC2_IAM_INFO, token:) + # Expected format: arn:aws:iam:::instance-profile/ + role_name = iam_info["InstanceProfileArn"].split("/").last + ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name, token:) + end - iam_info = ec2_metadata_request(EC2_IAM_INFO, token) + def ec2_metadata_credentials_imds_v1 + iam_info = ec2_metadata_request(EC2_IAM_INFO, token: nil) # Expected format: arn:aws:iam:::instance-profile/ role_name = iam_info["InstanceProfileArn"].split("/").last - ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name, token) + ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name, token: nil) end - def ec2_metadata_request(url, token) + def ec2_metadata_request(url, token:) request = ec2_iam_request(Gem::URI(url), Gem::Net::HTTP::Get) response = request.fetch do |req| - req.add_field "X-aws-ec2-metadata-token", token + if token + req.add_field "X-aws-ec2-metadata-token", token + end end case response @@ -185,11 +209,8 @@ def ec2_metadata_token end def ec2_iam_request(uri, verb) - @request_pool ||= {} - @request_pool[uri] ||= create_request_pool(uri) - pool = @request_pool[uri] - - Gem::Request.new(uri, verb, nil, pool) + @request_pool ||= create_request_pool(uri) + Gem::Request.new(uri, verb, nil, @request_pool) end def create_request_pool(uri) diff --git a/test/rubygems/test_gem_remote_fetcher_s3.rb b/test/rubygems/test_gem_remote_fetcher_s3.rb index 6c6f847b092304..664facd3def681 100644 --- a/test/rubygems/test_gem_remote_fetcher_s3.rb +++ b/test/rubygems/test_gem_remote_fetcher_s3.rb @@ -9,7 +9,6 @@ class TestGemRemoteFetcherS3 < Gem::TestCase include Gem::DefaultUserInteraction class FakeGemRequest < Gem::Request - attr_reader :last_request, :uri # Override perform_request to stub things @@ -52,12 +51,13 @@ def ec2_iam_request(uri, verb) case uri.to_s when "http://169.254.169.254/latest/api/token" - res = Gem::Net::HTTPOK.new nil, 200, nil - def res.body - "mysecrettoken" + if $imdsv2_token_failure + res = Gem::Net::HTTPUnauthorized.new nil, 401, nil + def res.body = "you got a 401! panic!" + else + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body = "mysecrettoken" end - fake_s3_request.set_response(res) - when "http://169.254.169.254/latest/meta-data/iam/info" res = Gem::Net::HTTPOK.new nil, 200, nil def res.body @@ -70,33 +70,26 @@ def res.body } JSON end - fake_s3_request.set_response(res) when "http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole" res = Gem::Net::HTTPOK.new nil, 200, nil - def res.body - $instance_profile - end - fake_s3_request.set_response(res) - + def res.body = $instance_profile else raise "Unexpected request to #{uri}" end + fake_s3_request.set_response(res) fake_s3_request end end class FakeGemFetcher < Gem::RemoteFetcher - attr_reader :fetched_uri, :last_s3_uri_signer def request(uri, request_class, last_modified = nil) @fetched_uri = uri res = Gem::Net::HTTPOK.new nil, 200, nil - def res.body - "success" - end + def res.body = "success" res end @@ -132,10 +125,33 @@ def assert_fetched_s3_with_imds_v2 assert_equal(expected.strip, recent_aws_query_logs.strip) end - def assert_fetch_s3(url:, signature:, token: nil, region: "us-east-1", instance_profile_json: nil, method: "GET") - @fetcher = FakeGemFetcher.new nil + def assert_fetched_s3_with_imds_v1 + # Three API requests: + # 1. Get the token (which fails) + # 2. Lookup profile details without token + # 3. Query the credentials without token + expected = <<~TEXT + PUT http://169.254.169.254/latest/api/token + x-aws-ec2-metadata-token-ttl-seconds=60 + GET http://169.254.169.254/latest/meta-data/iam/info + GET http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole + TEXT + recent_aws_query_logs = @fetcher.last_s3_uri_signer.recent_aws_query_logs + assert_equal(expected.strip, recent_aws_query_logs.strip) + end + + def with_imds_v2_failure + $imdsv2_token_failure = true + yield(fetcher) + ensure + $imdsv2_token_failure = nil + end + + def assert_fetch_s3(url:, signature:, token: nil, region: "us-east-1", instance_profile_json: nil, fetcher: nil, method: "GET") + @fetcher = fetcher || FakeGemFetcher.new(nil) $instance_profile = instance_profile_json - res = fetcher.fetch_s3 Gem::URI.parse(url), nil, (method == "HEAD") + res = @fetcher.fetch_s3 Gem::URI.parse(url), nil, (method == "HEAD") + $imdsv2_token_failure ||= nil assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T051941Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", @fetcher.fetched_uri.to_s if method == "HEAD" @@ -373,14 +389,16 @@ def test_fetch_s3_instance_profile_creds_with_fallback url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3( - url: url, - signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", - token: "testtoken", - region: "us-east-1", - instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' - ) - assert_fetched_s3_with_imds_v2 + with_imds_v2_failure do + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken", + region: "us-east-1", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' + ) + assert_fetched_s3_with_imds_v1 + end end ensure Gem.configuration[:s3_source] = nil From 374f7dbcbbf797cb3e5a1460140981d811bc7c10 Mon Sep 17 00:00:00 2001 From: pjsk Date: Mon, 16 Jun 2025 17:08:13 -0700 Subject: [PATCH 5/7] [rubygems/rubygems] removed global variables https://github.com/rubygems/rubygems/commit/42c5947dbe --- test/rubygems/test_gem_remote_fetcher_s3.rb | 40 +++++++-------------- 1 file changed, 12 insertions(+), 28 deletions(-) diff --git a/test/rubygems/test_gem_remote_fetcher_s3.rb b/test/rubygems/test_gem_remote_fetcher_s3.rb index 664facd3def681..8c9d363f65163e 100644 --- a/test/rubygems/test_gem_remote_fetcher_s3.rb +++ b/test/rubygems/test_gem_remote_fetcher_s3.rb @@ -23,6 +23,10 @@ def set_response(response) end class FakeS3URISigner < Gem::S3URISigner + class << self + attr_accessor :should_fail, :instance_profile + end + # Convenience method to output the recent aws iam queries made in tests # this outputs the verb, path, and any non-generic headers def recent_aws_query_logs @@ -51,7 +55,7 @@ def ec2_iam_request(uri, verb) case uri.to_s when "http://169.254.169.254/latest/api/token" - if $imdsv2_token_failure + if FakeS3URISigner.should_fail res = Gem::Net::HTTPUnauthorized.new nil, 401, nil def res.body = "you got a 401! panic!" else @@ -73,7 +77,7 @@ def res.body when "http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole" res = Gem::Net::HTTPOK.new nil, 200, nil - def res.body = $instance_profile + def res.body = FakeS3URISigner.instance_profile else raise "Unexpected request to #{uri}" end @@ -141,46 +145,26 @@ def assert_fetched_s3_with_imds_v1 end def with_imds_v2_failure - $imdsv2_token_failure = true + FakeS3URISigner.should_fail = true yield(fetcher) ensure - $imdsv2_token_failure = nil + FakeS3URISigner.should_fail = false end def assert_fetch_s3(url:, signature:, token: nil, region: "us-east-1", instance_profile_json: nil, fetcher: nil, method: "GET") + FakeS3URISigner.instance_profile = instance_profile_json + @fetcher = fetcher || FakeGemFetcher.new(nil) - $instance_profile = instance_profile_json res = @fetcher.fetch_s3 Gem::URI.parse(url), nil, (method == "HEAD") - $imdsv2_token_failure ||= nil - + assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T051941Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", @fetcher.fetched_uri.to_s if method == "HEAD" assert_equal 200, res.code else assert_equal "success", res end - - # Validation for EC2 IAM signing - if Gem.configuration[:s3_source]&.dig("my-bucket", :provider) == "instance_profile" - # Three API requests: - # 1. Get the token - # 2. Lookup profile details - # 3. Query the credentials - expected = <<~TEXT - PUT http://169.254.169.254/latest/api/token - x-aws-ec2-metadata-token-ttl-seconds=60 - GET http://169.254.169.254/latest/meta-data/iam/info - x-aws-ec2-metadata-token=mysecrettoken - GET http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole - x-aws-ec2-metadata-token=mysecrettoken - TEXT - recent_aws_query_logs = @fetcher.last_s3_uri_signer.recent_aws_query_logs - assert_equal(expected.strip, recent_aws_query_logs.strip) - else - assert_equal("", @fetcher.last_s3_uri_signer.recent_aws_query_logs) - end ensure - $instance_profile = nil + FakeS3URISigner.instance_profile = nil end def test_fetch_s3_config_creds From fe3ed3e7f3486e9bb8b1583ecb4e41efc882e4d3 Mon Sep 17 00:00:00 2001 From: pjsk Date: Mon, 16 Jun 2025 18:21:47 -0700 Subject: [PATCH 6/7] [rubygems/rubygems] Update tests to respect token for where v2 and v1 are invoked https://github.com/rubygems/rubygems/commit/261315e399 --- test/rubygems/test_gem_remote_fetcher_s3.rb | 60 +++++++++++---------- 1 file changed, 31 insertions(+), 29 deletions(-) diff --git a/test/rubygems/test_gem_remote_fetcher_s3.rb b/test/rubygems/test_gem_remote_fetcher_s3.rb index 8c9d363f65163e..4a5acc5a8666d4 100644 --- a/test/rubygems/test_gem_remote_fetcher_s3.rb +++ b/test/rubygems/test_gem_remote_fetcher_s3.rb @@ -24,7 +24,7 @@ def set_response(response) class FakeS3URISigner < Gem::S3URISigner class << self - attr_accessor :should_fail, :instance_profile + attr_accessor :return_token, :instance_profile end # Convenience method to output the recent aws iam queries made in tests @@ -55,12 +55,12 @@ def ec2_iam_request(uri, verb) case uri.to_s when "http://169.254.169.254/latest/api/token" - if FakeS3URISigner.should_fail + if FakeS3URISigner.return_token.nil? res = Gem::Net::HTTPUnauthorized.new nil, 401, nil def res.body = "you got a 401! panic!" else res = Gem::Net::HTTPOK.new nil, 200, nil - def res.body = "mysecrettoken" + def res.body = FakeS3URISigner.return_token end when "http://169.254.169.254/latest/meta-data/iam/info" res = Gem::Net::HTTPOK.new nil, 200, nil @@ -112,7 +112,7 @@ def setup @a1.loaded_from = File.join(@gemhome, "specifications", @a1.full_name) end - def assert_fetched_s3_with_imds_v2 + def assert_fetched_s3_with_imds_v2(expected_token) # Three API requests: # 1. Get the token # 2. Lookup profile details @@ -121,9 +121,9 @@ def assert_fetched_s3_with_imds_v2 PUT http://169.254.169.254/latest/api/token x-aws-ec2-metadata-token-ttl-seconds=60 GET http://169.254.169.254/latest/meta-data/iam/info - x-aws-ec2-metadata-token=mysecrettoken + x-aws-ec2-metadata-token=#{expected_token} GET http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole - x-aws-ec2-metadata-token=mysecrettoken + x-aws-ec2-metadata-token=#{expected_token} TEXT recent_aws_query_logs = @fetcher.last_s3_uri_signer.recent_aws_query_logs assert_equal(expected.strip, recent_aws_query_logs.strip) @@ -153,10 +153,11 @@ def with_imds_v2_failure def assert_fetch_s3(url:, signature:, token: nil, region: "us-east-1", instance_profile_json: nil, fetcher: nil, method: "GET") FakeS3URISigner.instance_profile = instance_profile_json - + FakeS3URISigner.return_token = token + @fetcher = fetcher || FakeGemFetcher.new(nil) res = @fetcher.fetch_s3 Gem::URI.parse(url), nil, (method == "HEAD") - + assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T051941Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", @fetcher.fetched_uri.to_s if method == "HEAD" assert_equal 200, res.code @@ -165,6 +166,7 @@ def assert_fetch_s3(url:, signature:, token: nil, region: "us-east-1", instance_ end ensure FakeS3URISigner.instance_profile = nil + FakeS3URISigner.return_token = nil end def test_fetch_s3_config_creds @@ -175,7 +177,7 @@ def test_fetch_s3_config_creds Time.stub :now, Time.at(1_561_353_581) do assert_fetch_s3( url: url, - signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", ) end ensure @@ -195,9 +197,9 @@ def test_fetch_s3_head_request assert_fetch_s3( url: url, - signature: "a3c6cf9a2db62e85f4e57f8fc8ac8b5ff5c1fdd4aeef55935d05e05174d9c885", - token: token, - region: region, + signature: "a3c6cf9a2db62e85f4e57f8fc8ac8b5ff5c1fdd4aeef55935d05e05174d9c885", + token: token, + region: region, instance_profile_json: instance_profile_json, method: method ) @@ -317,11 +319,12 @@ def test_fetch_s3_instance_profile_creds Time.stub :now, Time.at(1_561_353_581) do assert_fetch_s3( url: url, - signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", + signature: "da82e098bdaed0d3087047670efc98eaadc20559a473b5eac8d70190d2a9e8fd", region: "us-east-1", - instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + token: "mysecrettoken", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "mysecrettoken"}' ) - assert_fetched_s3_with_imds_v2 + assert_fetched_s3_with_imds_v2("mysecrettoken") end ensure Gem.configuration[:s3_source] = nil @@ -336,11 +339,12 @@ def test_fetch_s3_instance_profile_creds_with_region Time.stub :now, Time.at(1_561_353_581) do assert_fetch_s3( url: url, - signature: "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", + signature: "532960594dbfe31d1bbfc0e8e7a666c3cbdd8b00a143774da51b7f920704afd2", region: "us-west-2", - instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + token: "mysecrettoken", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "mysecrettoken"}' ) - assert_fetched_s3_with_imds_v2 + assert_fetched_s3_with_imds_v2("mysecrettoken") end ensure Gem.configuration[:s3_source] = nil @@ -360,7 +364,7 @@ def test_fetch_s3_instance_profile_creds_with_token region: "us-east-1", instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' ) - assert_fetched_s3_with_imds_v2 + assert_fetched_s3_with_imds_v2("testtoken") end ensure Gem.configuration[:s3_source] = nil @@ -373,16 +377,14 @@ def test_fetch_s3_instance_profile_creds_with_fallback url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - with_imds_v2_failure do - assert_fetch_s3( - url: url, - signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", - token: "testtoken", - region: "us-east-1", - instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' - ) - assert_fetched_s3_with_imds_v1 - end + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", + token: nil, + region: "us-east-1", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + ) + assert_fetched_s3_with_imds_v1 end ensure Gem.configuration[:s3_source] = nil From e60e1952a4bed328983b15918da5354246bcf320 Mon Sep 17 00:00:00 2001 From: Stan Lo Date: Wed, 6 Aug 2025 03:52:59 +0100 Subject: [PATCH 7/7] ZJIT: Fix `Kernel#Float`'s annotation (#14123) As pointed out in https://github.com/ruby/ruby/pull/14078#discussion_r2255427676, the return type should be `Float` instead. --- zjit/src/cruby_methods.rs | 2 +- zjit/src/hir.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index 8d1548f92b1717..c9ebcebc86d81b 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -174,7 +174,7 @@ pub fn init() -> Annotations { annotate!(rb_cNilClass, "nil?", types::TrueClass, no_gc, leaf, elidable); annotate!(rb_mKernel, "nil?", types::FalseClass, no_gc, leaf, elidable); - annotate_builtin!(rb_mKernel, "Float", types::Flonum); + annotate_builtin!(rb_mKernel, "Float", types::Float); annotate_builtin!(rb_mKernel, "Integer", types::Integer); annotate_builtin!(rb_mKernel, "class", types::Class, leaf); diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 635120eb80a8a0..1a67037ed35733 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -5011,9 +5011,9 @@ mod tests { assert_method_hir_with_opcode("Float", YARVINSN_opt_invokebuiltin_delegate_leave, expect![[r#" fn Float@: bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject, v3:BasicObject): - v6:Flonum = InvokeBuiltin rb_f_float, v0, v1, v2 + v6:Float = InvokeBuiltin rb_f_float, v0, v1, v2 Jump bb1(v0, v1, v2, v3, v6) - bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject, v12:Flonum): + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject, v12:Float): Return v12 "#]]); }