diff --git a/lib/rubygems/s3_uri_signer.rb b/lib/rubygems/s3_uri_signer.rb index 0d8e9e82851654..148cba38c47a9f 100644 --- a/lib/rubygems/s3_uri_signer.rb +++ b/lib/rubygems/s3_uri_signer.rb @@ -1,11 +1,14 @@ # frozen_string_literal: true require_relative "openssl" +require_relative "user_interaction" ## # S3URISigner implements AWS SigV4 for S3 Source to avoid a dependency on the aws-sdk-* gems # More on AWS SigV4: https://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html class Gem::S3URISigner + include Gem::UserInteraction + class ConfigurationError < Gem::Exception def initialize(message) super message @@ -147,17 +150,40 @@ def ec2_metadata_credentials_json require_relative "request/connection_pools" require "json" - iam_info = ec2_metadata_request(EC2_IAM_INFO) + # First try V2 fallback to V1 + res = nil + begin + res = ec2_metadata_credentials_imds_v2 + rescue InstanceProfileError + alert_warning "Unable to access ec2 credentials via IMDSv2, falling back to IMDSv1" + res = ec2_metadata_credentials_imds_v1 + end + res + end + + def ec2_metadata_credentials_imds_v2 + token = ec2_metadata_token + iam_info = ec2_metadata_request(EC2_IAM_INFO, token:) # Expected format: arn:aws:iam:::instance-profile/ role_name = iam_info["InstanceProfileArn"].split("/").last - ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name) + ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name, token:) end - def ec2_metadata_request(url) - uri = Gem::URI(url) - @request_pool ||= create_request_pool(uri) - request = Gem::Request.new(uri, Gem::Net::HTTP::Get, nil, @request_pool) - response = request.fetch + def ec2_metadata_credentials_imds_v1 + iam_info = ec2_metadata_request(EC2_IAM_INFO, token: nil) + # Expected format: arn:aws:iam:::instance-profile/ + role_name = iam_info["InstanceProfileArn"].split("/").last + ec2_metadata_request(EC2_IAM_SECURITY_CREDENTIALS + role_name, token: nil) + end + + def ec2_metadata_request(url, token:) + request = ec2_iam_request(Gem::URI(url), Gem::Net::HTTP::Get) + + response = request.fetch do |req| + if token + req.add_field "X-aws-ec2-metadata-token", token + end + end case response when Gem::Net::HTTPOK then @@ -167,6 +193,26 @@ def ec2_metadata_request(url) end end + def ec2_metadata_token + request = ec2_iam_request(Gem::URI(EC2_IAM_TOKEN), Gem::Net::HTTP::Put) + + response = request.fetch do |req| + req.add_field "X-aws-ec2-metadata-token-ttl-seconds", 60 + end + + case response + when Gem::Net::HTTPOK then + response.body + else + raise InstanceProfileError.new("Unable to fetch AWS metadata from #{uri}: #{response.message} #{response.code}") + end + end + + def ec2_iam_request(uri, verb) + @request_pool ||= create_request_pool(uri) + Gem::Request.new(uri, verb, nil, @request_pool) + end + def create_request_pool(uri) proxy_uri = Gem::Request.proxy_uri(Gem::Request.get_proxy_from_env(uri.scheme)) certs = Gem::Request.get_cert_files @@ -174,6 +220,7 @@ def create_request_pool(uri) end BASE64_URI_TRANSLATE = { "+" => "%2B", "/" => "%2F", "=" => "%3D", "\n" => "" }.freeze + EC2_IAM_TOKEN = "http://169.254.169.254/latest/api/token" EC2_IAM_INFO = "http://169.254.169.254/latest/meta-data/iam/info" EC2_IAM_SECURITY_CREDENTIALS = "http://169.254.169.254/latest/meta-data/iam/security-credentials/" end diff --git a/test/rubygems/test_gem_remote_fetcher_s3.rb b/test/rubygems/test_gem_remote_fetcher_s3.rb index e3aaa7a691f0a4..4a5acc5a8666d4 100644 --- a/test/rubygems/test_gem_remote_fetcher_s3.rb +++ b/test/rubygems/test_gem_remote_fetcher_s3.rb @@ -8,6 +8,100 @@ class TestGemRemoteFetcherS3 < Gem::TestCase include Gem::DefaultUserInteraction + class FakeGemRequest < Gem::Request + attr_reader :last_request, :uri + + # Override perform_request to stub things + def perform_request(request) + @last_request = request + @response + end + + def set_response(response) + @response = response + end + end + + class FakeS3URISigner < Gem::S3URISigner + class << self + attr_accessor :return_token, :instance_profile + end + + # Convenience method to output the recent aws iam queries made in tests + # this outputs the verb, path, and any non-generic headers + def recent_aws_query_logs + sreqs = @aws_iam_calls.map do |c| + r = c.last_request + s = +"#{r.method} #{c.uri}\n" + r.each_header do |key, v| + # Only include headers that start with x- + next unless key.start_with?("x-") + s << " #{key}=#{v}\n" + end + s + end + + sreqs.join("") + end + + def initialize(uri, method) + @aws_iam_calls = [] + super + end + + def ec2_iam_request(uri, verb) + fake_s3_request = FakeGemRequest.new(uri, verb, nil, nil) + @aws_iam_calls << fake_s3_request + + case uri.to_s + when "http://169.254.169.254/latest/api/token" + if FakeS3URISigner.return_token.nil? + res = Gem::Net::HTTPUnauthorized.new nil, 401, nil + def res.body = "you got a 401! panic!" + else + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body = FakeS3URISigner.return_token + end + when "http://169.254.169.254/latest/meta-data/iam/info" + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body + <<~JSON + { + "Code": "Success", + "LastUpdated": "2023-05-27:05:05", + "InstanceProfileArn": "arn:aws:iam::somesecretid:instance-profile/TestRole", + "InstanceProfileId": "SOMEPROFILEID" + } + JSON + end + + when "http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole" + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body = FakeS3URISigner.instance_profile + else + raise "Unexpected request to #{uri}" + end + + fake_s3_request.set_response(res) + fake_s3_request + end + end + + class FakeGemFetcher < Gem::RemoteFetcher + attr_reader :fetched_uri, :last_s3_uri_signer + + def request(uri, request_class, last_modified = nil) + @fetched_uri = uri + res = Gem::Net::HTTPOK.new nil, 200, nil + def res.body = "success" + res + end + + def s3_uri_signer(uri, method) + @last_s3_uri_signer = FakeS3URISigner.new(uri, method) + end + end + def setup super @@ -18,43 +112,61 @@ def setup @a1.loaded_from = File.join(@gemhome, "specifications", @a1.full_name) end - def assert_fetch_s3(url, signature, token=nil, region="us-east-1", instance_profile_json=nil, method="GET") - fetcher = Gem::RemoteFetcher.new nil - @fetcher = fetcher - $fetched_uri = nil - $instance_profile = instance_profile_json + def assert_fetched_s3_with_imds_v2(expected_token) + # Three API requests: + # 1. Get the token + # 2. Lookup profile details + # 3. Query the credentials + expected = <<~TEXT + PUT http://169.254.169.254/latest/api/token + x-aws-ec2-metadata-token-ttl-seconds=60 + GET http://169.254.169.254/latest/meta-data/iam/info + x-aws-ec2-metadata-token=#{expected_token} + GET http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole + x-aws-ec2-metadata-token=#{expected_token} + TEXT + recent_aws_query_logs = @fetcher.last_s3_uri_signer.recent_aws_query_logs + assert_equal(expected.strip, recent_aws_query_logs.strip) + end - def fetcher.request(uri, request_class, last_modified = nil) - $fetched_uri = uri - res = Gem::Net::HTTPOK.new nil, 200, nil - def res.body - "success" - end - res - end + def assert_fetched_s3_with_imds_v1 + # Three API requests: + # 1. Get the token (which fails) + # 2. Lookup profile details without token + # 3. Query the credentials without token + expected = <<~TEXT + PUT http://169.254.169.254/latest/api/token + x-aws-ec2-metadata-token-ttl-seconds=60 + GET http://169.254.169.254/latest/meta-data/iam/info + GET http://169.254.169.254/latest/meta-data/iam/security-credentials/TestRole + TEXT + recent_aws_query_logs = @fetcher.last_s3_uri_signer.recent_aws_query_logs + assert_equal(expected.strip, recent_aws_query_logs.strip) + end - def fetcher.s3_uri_signer(uri, method) - require "json" - s3_uri_signer = Gem::S3URISigner.new(uri, method) - def s3_uri_signer.ec2_metadata_credentials_json - JSON.parse($instance_profile) - end - # Running sign operation to make sure uri.query is not mutated - s3_uri_signer.sign - raise "URI query is not empty: #{uri.query}" unless uri.query.nil? - s3_uri_signer - end + def with_imds_v2_failure + FakeS3URISigner.should_fail = true + yield(fetcher) + ensure + FakeS3URISigner.should_fail = false + end - res = fetcher.fetch_s3 Gem::URI.parse(url), nil, (method == "HEAD") + def assert_fetch_s3(url:, signature:, token: nil, region: "us-east-1", instance_profile_json: nil, fetcher: nil, method: "GET") + FakeS3URISigner.instance_profile = instance_profile_json + FakeS3URISigner.return_token = token - assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T051941Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", $fetched_uri.to_s + @fetcher = fetcher || FakeGemFetcher.new(nil) + res = @fetcher.fetch_s3 Gem::URI.parse(url), nil, (method == "HEAD") + + assert_equal "https://my-bucket.s3.#{region}.amazonaws.com/gems/specs.4.8.gz?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=testuser%2F20190624%2F#{region}%2Fs3%2Faws4_request&X-Amz-Date=20190624T051941Z&X-Amz-Expires=86400#{token ? "&X-Amz-Security-Token=" + token : ""}&X-Amz-SignedHeaders=host&X-Amz-Signature=#{signature}", @fetcher.fetched_uri.to_s if method == "HEAD" assert_equal 200, res.code else assert_equal "success", res end ensure - $fetched_uri = nil + FakeS3URISigner.instance_profile = nil + FakeS3URISigner.return_token = nil end def test_fetch_s3_config_creds @@ -63,7 +175,10 @@ def test_fetch_s3_config_creds } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", + ) end ensure Gem.configuration[:s3_source] = nil @@ -79,7 +194,15 @@ def test_fetch_s3_head_request region = "us-east-1" instance_profile_json = nil method = "HEAD" - assert_fetch_s3 url, "a3c6cf9a2db62e85f4e57f8fc8ac8b5ff5c1fdd4aeef55935d05e05174d9c885", token, region, instance_profile_json, method + + assert_fetch_s3( + url: url, + signature: "a3c6cf9a2db62e85f4e57f8fc8ac8b5ff5c1fdd4aeef55935d05e05174d9c885", + token: token, + region: region, + instance_profile_json: instance_profile_json, + method: method + ) end ensure Gem.configuration[:s3_source] = nil @@ -91,7 +214,11 @@ def test_fetch_s3_config_creds_with_region } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", nil, "us-west-2" + assert_fetch_s3( + url: url, + signature: "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", + region: "us-west-2" + ) end ensure Gem.configuration[:s3_source] = nil @@ -103,7 +230,11 @@ def test_fetch_s3_config_creds_with_token } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", "testtoken" + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken" + ) end ensure Gem.configuration[:s3_source] = nil @@ -118,7 +249,10 @@ def test_fetch_s3_env_creds } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + ) end ensure ENV.each_key {|key| ENV.delete(key) if key.start_with?("AWS") } @@ -134,7 +268,12 @@ def test_fetch_s3_env_creds_with_region } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", nil, "us-west-2" + assert_fetch_s3( + url: url, + signature: "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", + token: nil, + region: "us-west-2" + ) end ensure ENV.each_key {|key| ENV.delete(key) if key.start_with?("AWS") } @@ -150,7 +289,11 @@ def test_fetch_s3_env_creds_with_token } url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", "testtoken" + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken" + ) end ensure ENV.each_key {|key| ENV.delete(key) if key.start_with?("AWS") } @@ -160,7 +303,10 @@ def test_fetch_s3_env_creds_with_token def test_fetch_s3_url_creds url = "s3://testuser:testpass@my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c" + ) end end @@ -171,8 +317,14 @@ def test_fetch_s3_instance_profile_creds url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", nil, "us-east-1", - '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + assert_fetch_s3( + url: url, + signature: "da82e098bdaed0d3087047670efc98eaadc20559a473b5eac8d70190d2a9e8fd", + region: "us-east-1", + token: "mysecrettoken", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "mysecrettoken"}' + ) + assert_fetched_s3_with_imds_v2("mysecrettoken") end ensure Gem.configuration[:s3_source] = nil @@ -185,8 +337,14 @@ def test_fetch_s3_instance_profile_creds_with_region url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "ef07487bfd8e3ca594f8fc29775b70c0a0636f51318f95d4f12b2e6e1fd8c716", nil, "us-west-2", - '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + assert_fetch_s3( + url: url, + signature: "532960594dbfe31d1bbfc0e8e7a666c3cbdd8b00a143774da51b7f920704afd2", + region: "us-west-2", + token: "mysecrettoken", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "mysecrettoken"}' + ) + assert_fetched_s3_with_imds_v2("mysecrettoken") end ensure Gem.configuration[:s3_source] = nil @@ -199,14 +357,40 @@ def test_fetch_s3_instance_profile_creds_with_token url = "s3://my-bucket/gems/specs.4.8.gz" Time.stub :now, Time.at(1_561_353_581) do - assert_fetch_s3 url, "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", "testtoken", "us-east-1", - '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' + assert_fetch_s3( + url: url, + signature: "e709338735f9077edf8f6b94b247171c266a9605975e08e4a519a123c3322625", + token: "testtoken", + region: "us-east-1", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass", "Token": "testtoken"}' + ) + assert_fetched_s3_with_imds_v2("testtoken") + end + ensure + Gem.configuration[:s3_source] = nil + end + + def test_fetch_s3_instance_profile_creds_with_fallback + Gem.configuration[:s3_source] = { + "my-bucket" => { provider: "instance_profile" }, + } + + url = "s3://my-bucket/gems/specs.4.8.gz" + Time.stub :now, Time.at(1_561_353_581) do + assert_fetch_s3( + url: url, + signature: "b5cb80c1301f7b1c50c4af54f1f6c034f80b56d32f000a855f0a903dc5a8413c", + token: nil, + region: "us-east-1", + instance_profile_json: '{"AccessKeyId": "testuser", "SecretAccessKey": "testpass"}' + ) + assert_fetched_s3_with_imds_v1 end ensure Gem.configuration[:s3_source] = nil end - def refute_fetch_s3(url, expected_message) + def refute_fetch_s3(url:, expected_message:) fetcher = Gem::RemoteFetcher.new nil @fetcher = fetcher @@ -219,7 +403,7 @@ def refute_fetch_s3(url, expected_message) def test_fetch_s3_no_source_key url = "s3://my-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "no s3_source key exists in .gemrc" + refute_fetch_s3(url: url, expected_message: "no s3_source key exists in .gemrc") end def test_fetch_s3_no_host @@ -228,7 +412,7 @@ def test_fetch_s3_no_host } url = "s3://other-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "no key for host other-bucket in s3_source in .gemrc" + refute_fetch_s3(url: url, expected_message: "no key for host other-bucket in s3_source in .gemrc") ensure Gem.configuration[:s3_source] = nil end @@ -237,7 +421,7 @@ def test_fetch_s3_no_id Gem.configuration[:s3_source] = { "my-bucket" => { secret: "testpass" } } url = "s3://my-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "s3_source for my-bucket missing id or secret" + refute_fetch_s3(url: url, expected_message: "s3_source for my-bucket missing id or secret") ensure Gem.configuration[:s3_source] = nil end @@ -246,7 +430,7 @@ def test_fetch_s3_no_secret Gem.configuration[:s3_source] = { "my-bucket" => { id: "testuser" } } url = "s3://my-bucket/gems/specs.4.8.gz" - refute_fetch_s3 url, "s3_source for my-bucket missing id or secret" + refute_fetch_s3(url: url, expected_message: "s3_source for my-bucket missing id or secret") ensure Gem.configuration[:s3_source] = nil end diff --git a/zjit/src/cruby_methods.rs b/zjit/src/cruby_methods.rs index 8d1548f92b1717..c9ebcebc86d81b 100644 --- a/zjit/src/cruby_methods.rs +++ b/zjit/src/cruby_methods.rs @@ -174,7 +174,7 @@ pub fn init() -> Annotations { annotate!(rb_cNilClass, "nil?", types::TrueClass, no_gc, leaf, elidable); annotate!(rb_mKernel, "nil?", types::FalseClass, no_gc, leaf, elidable); - annotate_builtin!(rb_mKernel, "Float", types::Flonum); + annotate_builtin!(rb_mKernel, "Float", types::Float); annotate_builtin!(rb_mKernel, "Integer", types::Integer); annotate_builtin!(rb_mKernel, "class", types::Class, leaf); diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 635120eb80a8a0..1a67037ed35733 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -5011,9 +5011,9 @@ mod tests { assert_method_hir_with_opcode("Float", YARVINSN_opt_invokebuiltin_delegate_leave, expect![[r#" fn Float@: bb0(v0:BasicObject, v1:BasicObject, v2:BasicObject, v3:BasicObject): - v6:Flonum = InvokeBuiltin rb_f_float, v0, v1, v2 + v6:Float = InvokeBuiltin rb_f_float, v0, v1, v2 Jump bb1(v0, v1, v2, v3, v6) - bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject, v12:Flonum): + bb1(v8:BasicObject, v9:BasicObject, v10:BasicObject, v11:BasicObject, v12:Float): Return v12 "#]]); }