From b306424986d432eaa08b77da074704391006890e Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 28 Sep 2020 13:41:36 -0700 Subject: [PATCH 01/17] Add basic Credentials wrapper + Sigv base interface --- .rubocop.yml | 3 +- gems/aws-crt-auth/lib/aws-crt-auth.rb | 2 + .../lib/aws-crt-auth/credentials.rb | 95 +++++++++++++++ gems/aws-crt-auth/lib/aws-crt-auth/signer.rb | 37 ++++++ gems/aws-crt-auth/spec/credentials_spec.rb | 112 ++++++++++++++++++ gems/aws-crt-auth/spec/signer_spec.rb | 9 ++ gems/aws-crt-auth/spec/spec_helper.rb | 1 + gems/aws-crt/lib/aws-crt/native.rb | 10 ++ gems/aws-crt/native/src/api.h | 13 ++ gems/aws-crt/native/src/credentials.c | 46 +++++++ 10 files changed, 327 insertions(+), 1 deletion(-) create mode 100644 gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb create mode 100644 gems/aws-crt-auth/lib/aws-crt-auth/signer.rb create mode 100644 gems/aws-crt-auth/spec/credentials_spec.rb create mode 100644 gems/aws-crt-auth/spec/signer_spec.rb create mode 100644 gems/aws-crt/native/src/credentials.c diff --git a/.rubocop.yml b/.rubocop.yml index cccf698..770e4ca 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -19,9 +19,10 @@ Metrics/BlockLength: Exclude: - 'gems/aws-crt/Rakefile' - '**/*.rake' - - 'gems/aws-crt/spec/**/*.rb' + - 'gems/**/spec/**/*.rb' Metrics/MethodLength: + Max: 25 Exclude: - 'gems/aws-crt/ext/compile.rb' diff --git a/gems/aws-crt-auth/lib/aws-crt-auth.rb b/gems/aws-crt-auth/lib/aws-crt-auth.rb index 46ba442..33d01cd 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth.rb @@ -1,6 +1,8 @@ # frozen_string_literal: true require 'aws-crt' +require_relative 'aws-crt-auth/signer' +require_relative 'aws-crt-auth/credentials' module Aws module Crt diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb new file mode 100644 index 0000000..e876c6b --- /dev/null +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Aws + module Crt + module Auth + # Utility class for Credentials. + class Credentials + # @param [String] access_key_id + # @param [String] secret_access_key + # @param [String] session_token (nil) + def initialize(access_key_id, secret_access_key, + session_token = nil, expiration = nil) + if !access_key_id || access_key_id.empty? + raise ArgumentError, 'access_key_id must be set' + end + + if !secret_access_key || secret_access_key.empty? + raise ArgumentError, 'secret_access_key must be set' + end + + native = Aws::Crt.call do + # -1 will give UINT max + Aws::Crt::Native.credentials_new( + access_key_id, + secret_access_key, + session_token, + expiration&.to_i || -1 + ) + end + + @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) + end + + # @return [String, nil] + def access_key_id + Aws::Crt::Native.credentials_get_access_key_id(@native) if @native + end + + # @return [String, nil] + def secret_access_key + Aws::Crt::Native.credentials_get_secret_access_key(@native) if @native + end + + # @return [String, nil] + def session_token + Aws::Crt::Native.credentials_get_session_token(@native) if @native + end + + # @return [Time,nil] + def expiration + if @native + # TODO: Return nil if exp is UINT64_MAX? + Time.at(Aws::Crt::Native.credentials_get_expiration(@native)) + end + end + + # @return [Credentials] + def credentials + self + end + + # @return [Boolean] Returns `true` if the access key id and secret + # access key are both set. + def set? + !@native.nil? && + !access_key_id.nil? && + !access_key_id.empty? && + !secret_access_key.nil? && + !secret_access_key.empty? + end + + # Removing the secret access key from the default inspect string. + # @api private + def inspect + "#<#{self.class.name} access_key_id=#{access_key_id.inspect}>" + end + + # Immediately release this instance's attachment to the underlying + # resources, without waiting for the garbage collector. + # Note that underlying resources will remain alive until nothing + # else is using them. + def release + return unless @native + + @native.free + @native = nil + end + + def self.on_release(native) + Aws::Crt::Native.event_loop_group_release(native) + end + end + end + end +end diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/signer.rb b/gems/aws-crt-auth/lib/aws-crt-auth/signer.rb new file mode 100644 index 0000000..da111d3 --- /dev/null +++ b/gems/aws-crt-auth/lib/aws-crt-auth/signer.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Aws + module Crt + module Auth + # Utility class for creating AWS signature version 4 signature. + class Signer + def initialize(options = {}) + @service = extract_service(options) + @region = extract_region(options) + @credentials_provider = extract_credentials_provider(options) + @unsigned_headers = Set.new((options.fetch(:unsigned_headers, [])) + .map(&:downcase)) + @unsigned_headers << 'authorization' + @unsigned_headers << 'x-amzn-trace-id' + @unsigned_headers << 'expect' + %i[uri_escape_path apply_checksum_header].each do |opt| + instance_variable_set("@#{opt}", + options.key?(opt) ? options[:opt] : true) + end + end + + def sign_request(request) + # TODO + end + + def sign_event(prior_signature, payload, encoder) + # TODO + end + + def presign_url(options) + # TODO + end + end + end + end +end diff --git a/gems/aws-crt-auth/spec/credentials_spec.rb b/gems/aws-crt-auth/spec/credentials_spec.rb new file mode 100644 index 0000000..5f325d3 --- /dev/null +++ b/gems/aws-crt-auth/spec/credentials_spec.rb @@ -0,0 +1,112 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'weakref' + +module Aws + module Crt + module Auth + UINT64_MAX = 18_446_744_073_709_551_615 + + describe Credentials do + describe '#initilize' do + it 'raises an ArgumentError when missing access_key_id' do + expect { Credentials.new(nil, 'secret') } + .to raise_error(ArgumentError) + end + + it 'raises an ArgumentError when missing secret_access_key' do + expect { Credentials.new('akid', nil) } + .to raise_error(ArgumentError) + end + + it 'defaults the session token to nil' do + expect(Credentials.new('akid', 'secret').session_token).to be nil + end + + it 'defaults the expiration to UINT64_MAX' do + expect(Credentials.new('akid', 'secret').expiration.to_i) + .to eq UINT64_MAX + end + + it 'accepts a Time for expiration' do + exp = Time.now + creds = Credentials.new('akid', 'secret', 'token', exp) + expect(creds.expiration.to_i).to eq exp.to_i + end + + it 'accepts an epoch (integer) for expiration' do + exp = Time.now + creds = Credentials.new('akid', 'secret', 'token', exp.to_i) + expect(creds.expiration.to_i).to eq exp.to_i + end + end + + describe 'accessors' do + let(:exp) { Time.now } + let(:creds) { Credentials.new('akid', 'secret', 'token', exp) } + + it 'provides access to the access key id' do + expect(creds.access_key_id).to eq('akid') + end + + it 'provides access to the secret access key' do + expect(creds.secret_access_key).to eq('secret') + end + + it 'provides access to the session token' do + expect(creds.session_token).to eq('token') + end + + it 'provides access to the expiration' do + expect(creds.expiration.to_i).to eq exp.to_i + end + end + + describe '#set?' do + it 'returns true when the key and secret are both non nil values' do + expect(Credentials.new('akid', 'secret').set?).to be(true) + end + + it 'returns false after the credentials have been released' do + creds = Credentials.new('akid', 'secret') + creds.release + expect(creds.set?).to be(false) + end + end + + describe '#inspect' do + let(:creds) { Credentials.new('akid', 'secret', 'token') } + + it 'does not include the secret_access_key' do + expect(creds.inspect).not_to include 'secret' + end + end + + describe '.on_release' do + it 'cleans up with release' do + creds = Credentials.new('akid', 'secret') + expect(creds).to_not be_nil + + creds.release + check_for_clean_shutdown + end + + if garbage_collect_is_immediate? + it 'cleans up with GC' do + creds = Credentials.new('akid', 'secret') + weakref = WeakRef.new(creds) + expect(weakref.weakref_alive?).to be true + + # force cleanup via GC + creds = nil # rubocop:disable Lint/UselessAssignment + ObjectSpace.garbage_collect + expect(weakref.weakref_alive?).to be_falsey + check_for_clean_shutdown + end + end + end + end + end + end +end diff --git a/gems/aws-crt-auth/spec/signer_spec.rb b/gems/aws-crt-auth/spec/signer_spec.rb new file mode 100644 index 0000000..253e7d7 --- /dev/null +++ b/gems/aws-crt-auth/spec/signer_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' + +require 'tempfile' +require 'base64' + +describe Aws::Crt::Auth::Signer do +end diff --git a/gems/aws-crt-auth/spec/spec_helper.rb b/gems/aws-crt-auth/spec/spec_helper.rb index 1f72a1f..9f4b41c 100644 --- a/gems/aws-crt-auth/spec/spec_helper.rb +++ b/gems/aws-crt-auth/spec/spec_helper.rb @@ -5,4 +5,5 @@ # use the local version of aws-crt $LOAD_PATH.unshift File.expand_path('../../aws-crt/lib', __dir__) +require_relative '../../aws-crt/spec/spec_helper' require 'aws-crt-auth' diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index 7ed5562..4718102 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -18,9 +18,19 @@ module Native attach_function :error_debug_str, :aws_crt_error_debug_str, [:int], :string attach_function :reset_error, :aws_crt_reset_error, [], :void attach_function :global_thread_creator_shutdown_wait_for, :aws_crt_global_thread_creator_shutdown_wait_for, [:uint32], :int + # IO API attach_function :event_loop_group_new, :aws_crt_event_loop_group_new, [:uint16], :pointer attach_function :event_loop_group_release, :aws_crt_event_loop_group_release, [:pointer], :void + + # Auth API + attach_function :credentials_new, :aws_crt_credentials_new, %i[string string string uint64], :pointer + attach_function :credentials_release, :aws_crt_credentials_release, [:pointer], :void + attach_function :credentials_get_access_key_id, :aws_crt_credentials_get_access_key_id, [:pointer], :string + attach_function :credentials_get_secret_access_key, :aws_crt_credentials_get_secret_access_key, [:pointer], :string + attach_function :credentials_get_session_token, :aws_crt_credentials_get_session_token, [:pointer], :string + attach_function :credentials_get_expiration, :aws_crt_credentials_get_expiration_timepoint_seconds, [:pointer], :uint64 + # Internal testing API attach_function :test_error, :aws_crt_test_error, [:int], :int attach_function :test_pointer_error, :aws_crt_test_pointer_error, [], :pointer diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index 910b0f0..6ae5a17 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -42,6 +42,19 @@ AWS_CRT_API void aws_crt_reset_error(void); AWS_CRT_API struct aws_event_loop_group *aws_crt_event_loop_group_new(uint16_t max_threads); AWS_CRT_API void aws_crt_event_loop_group_release(struct aws_event_loop_group *elg); +/* Auth */ +AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( + const char *access_key_id, + const char *secret_access_key, + const char *session_token, + uint64_t expiration_timepoint_seconds); +AWS_CRT_API void aws_crt_credentials_release(struct aws_credentials *credentials); +AWS_CRT_API const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials); +AWS_CRT_API const char *aws_crt_credentials_get_secret_access_key(struct aws_credentials *credentials); +AWS_CRT_API const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials); +AWS_CRT_API uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials); + + AWS_EXTERN_C_END #endif /* AWS_CRT_API_H */ diff --git a/gems/aws-crt/native/src/credentials.c b/gems/aws-crt/native/src/credentials.c new file mode 100644 index 0000000..e615963 --- /dev/null +++ b/gems/aws-crt/native/src/credentials.c @@ -0,0 +1,46 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include "crt.h" + +#include +#include + +AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( + const char *access_key_id, + const char *secret_access_key, + const char *session_token, + uint64_t expiration_timepoint_seconds) { + + struct aws_allocator *allocator = aws_crt_allocator(); + return aws_credentials_new_from_string(allocator, + aws_string_new_from_c_str(allocator, access_key_id), + aws_string_new_from_c_str(allocator, secret_access_key), + session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), + expiration_timepoint_seconds); + //TODO: Do these strings get cleaned up by the release of the credentials?? + // aws_byte_cursor_from_c_str(access_key_id), + // aws_byte_cursor_from_c_str(secret_access_key), + // aws_byte_cursor_from_c_str(session_token), +} + +const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials) { + return (char *)aws_credentials_get_access_key_id(credentials).ptr; +} + +const char *aws_crt_credentials_get_secret_access_key(struct aws_credentials *credentials) { + return (char *)aws_credentials_get_secret_access_key(credentials).ptr; +} + +const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials) { + return (char *)aws_credentials_get_session_token(credentials).ptr; +} + +uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials) { + return aws_credentials_get_expiration_timepoint_seconds(credentials); +} + +void aws_crt_credentials_release(struct aws_credentials *credentials) { + aws_credentials_release(credentials); +} From 33158d0f0b94d4745c3158bfb2bb97111a2b04ef Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 28 Sep 2020 13:58:04 -0700 Subject: [PATCH 02/17] Rubocop cleanups --- gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb | 8 ++++---- gems/aws-crt-auth/spec/credentials_spec.rb | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index e876c6b..beaadd7 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -48,10 +48,10 @@ def session_token # @return [Time,nil] def expiration - if @native - # TODO: Return nil if exp is UINT64_MAX? - Time.at(Aws::Crt::Native.credentials_get_expiration(@native)) - end + return unless @native + + # TODO: Return nil if exp is UINT64_MAX? + Time.at(Aws::Crt::Native.credentials_get_expiration(@native)) end # @return [Credentials] diff --git a/gems/aws-crt-auth/spec/credentials_spec.rb b/gems/aws-crt-auth/spec/credentials_spec.rb index 5f325d3..2670e4b 100644 --- a/gems/aws-crt-auth/spec/credentials_spec.rb +++ b/gems/aws-crt-auth/spec/credentials_spec.rb @@ -5,7 +5,7 @@ module Aws module Crt - module Auth + module Auth #:nodoc: UINT64_MAX = 18_446_744_073_709_551_615 describe Credentials do From 5356c0facb4d473adc346eb730045a60ade9bb35 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 28 Sep 2020 14:04:13 -0700 Subject: [PATCH 03/17] Clang format --- gems/aws-crt/native/src/api.h | 9 ++++---- gems/aws-crt/native/src/credentials.c | 32 +++++++++++++-------------- 2 files changed, 20 insertions(+), 21 deletions(-) diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index 6ae5a17..add4bbd 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -44,17 +44,16 @@ AWS_CRT_API void aws_crt_event_loop_group_release(struct aws_event_loop_group *e /* Auth */ AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( - const char *access_key_id, - const char *secret_access_key, - const char *session_token, - uint64_t expiration_timepoint_seconds); + const char *access_key_id, + const char *secret_access_key, + const char *session_token, + uint64_t expiration_timepoint_seconds); AWS_CRT_API void aws_crt_credentials_release(struct aws_credentials *credentials); AWS_CRT_API const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials); AWS_CRT_API const char *aws_crt_credentials_get_secret_access_key(struct aws_credentials *credentials); AWS_CRT_API const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials); AWS_CRT_API uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials); - AWS_EXTERN_C_END #endif /* AWS_CRT_API_H */ diff --git a/gems/aws-crt/native/src/credentials.c b/gems/aws-crt/native/src/credentials.c index e615963..a9ffba0 100644 --- a/gems/aws-crt/native/src/credentials.c +++ b/gems/aws-crt/native/src/credentials.c @@ -4,21 +4,21 @@ */ #include "crt.h" -#include #include +#include AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( - const char *access_key_id, - const char *secret_access_key, - const char *session_token, - uint64_t expiration_timepoint_seconds) { - - struct aws_allocator *allocator = aws_crt_allocator(); - return aws_credentials_new_from_string(allocator, - aws_string_new_from_c_str(allocator, access_key_id), - aws_string_new_from_c_str(allocator, secret_access_key), - session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), - expiration_timepoint_seconds); + const char *access_key_id, + const char *secret_access_key, + const char *session_token, + uint64_t expiration_timepoint_seconds) { + + struct aws_allocator *allocator = aws_crt_allocator(); + return aws_credentials_new_from_string(allocator, + aws_string_new_from_c_str(allocator, access_key_id), + aws_string_new_from_c_str(allocator, secret_access_key), + session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), + expiration_timepoint_seconds); //TODO: Do these strings get cleaned up by the release of the credentials?? // aws_byte_cursor_from_c_str(access_key_id), // aws_byte_cursor_from_c_str(secret_access_key), @@ -26,19 +26,19 @@ AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( } const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials) { - return (char *)aws_credentials_get_access_key_id(credentials).ptr; + return (char *)aws_credentials_get_access_key_id(credentials).ptr; } const char *aws_crt_credentials_get_secret_access_key(struct aws_credentials *credentials) { - return (char *)aws_credentials_get_secret_access_key(credentials).ptr; + return (char *)aws_credentials_get_secret_access_key(credentials).ptr; } const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials) { - return (char *)aws_credentials_get_session_token(credentials).ptr; + return (char *)aws_credentials_get_session_token(credentials).ptr; } uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials) { - return aws_credentials_get_expiration_timepoint_seconds(credentials); + return aws_credentials_get_expiration_timepoint_seconds(credentials); } void aws_crt_credentials_release(struct aws_credentials *credentials) { From 1f64ed8f94feb1f2b1c7517c905a014abe352102 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Mon, 28 Sep 2020 15:06:11 -0700 Subject: [PATCH 04/17] Fix cleanup issues (use correct function...) --- .../lib/aws-crt-auth/credentials.rb | 12 ++++++++++- gems/aws-crt-auth/spec/credentials_spec.rb | 2 +- gems/aws-crt/native/src/credentials.c | 21 +++++++++++-------- 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index beaadd7..2b7f882 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -18,6 +18,16 @@ def initialize(access_key_id, secret_access_key, raise ArgumentError, 'secret_access_key must be set' end + # access_key_id = FFI::MemoryPointer.from_string(access_key_id) + # access_key_id.autorelease = false + # secret_access_key = FFI::MemoryPointer.from_string(secret_access_key) + # secret_access_key.autorelease = false + # if session_token + # session_token = FFI::MemoryPointer.from_string(session_token) + # session_token.autorelease = false + # end + + native = Aws::Crt.call do # -1 will give UINT max Aws::Crt::Native.credentials_new( @@ -87,7 +97,7 @@ def release end def self.on_release(native) - Aws::Crt::Native.event_loop_group_release(native) + Aws::Crt::Native.credentials_release(native) end end end diff --git a/gems/aws-crt-auth/spec/credentials_spec.rb b/gems/aws-crt-auth/spec/credentials_spec.rb index 2670e4b..b057924 100644 --- a/gems/aws-crt-auth/spec/credentials_spec.rb +++ b/gems/aws-crt-auth/spec/credentials_spec.rb @@ -94,7 +94,7 @@ module Auth #:nodoc: if garbage_collect_is_immediate? it 'cleans up with GC' do - creds = Credentials.new('akid', 'secret') + creds = Credentials.new('akid', 'secret', 'session') weakref = WeakRef.new(creds) expect(weakref.weakref_alive?).to be true diff --git a/gems/aws-crt/native/src/credentials.c b/gems/aws-crt/native/src/credentials.c index a9ffba0..8fc1c23 100644 --- a/gems/aws-crt/native/src/credentials.c +++ b/gems/aws-crt/native/src/credentials.c @@ -14,15 +14,18 @@ AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( uint64_t expiration_timepoint_seconds) { struct aws_allocator *allocator = aws_crt_allocator(); - return aws_credentials_new_from_string(allocator, - aws_string_new_from_c_str(allocator, access_key_id), - aws_string_new_from_c_str(allocator, secret_access_key), - session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), - expiration_timepoint_seconds); - //TODO: Do these strings get cleaned up by the release of the credentials?? - // aws_byte_cursor_from_c_str(access_key_id), - // aws_byte_cursor_from_c_str(secret_access_key), - // aws_byte_cursor_from_c_str(session_token), + return aws_credentials_new( + allocator, + aws_byte_cursor_from_c_str(access_key_id), + aws_byte_cursor_from_c_str(secret_access_key), + aws_byte_cursor_from_c_str(session_token), + expiration_timepoint_seconds); +// return aws_credentials_new_from_string( +// allocator, +// aws_string_new_from_c_str(allocator, access_key_id), +// aws_string_new_from_c_str(allocator, secret_access_key), +// session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), +// expiration_timepoint_seconds); } const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials) { From a7a1acfbf5ee5152760c5b9856347988d3faca52 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 29 Sep 2020 09:00:57 -0700 Subject: [PATCH 05/17] Fix Jruby issue with UINT64 out of range for Time --- gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb | 8 ++++++-- gems/aws-crt-auth/spec/credentials_spec.rb | 6 +++--- gems/aws-crt/native/src/credentials.c | 12 ++++++------ 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index 2b7f882..a155fce 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -5,6 +5,8 @@ module Crt module Auth # Utility class for Credentials. class Credentials + UINT64_MAX = 18_446_744_073_709_551_615 + # @param [String] access_key_id # @param [String] secret_access_key # @param [String] session_token (nil) @@ -60,8 +62,10 @@ def session_token def expiration return unless @native - # TODO: Return nil if exp is UINT64_MAX? - Time.at(Aws::Crt::Native.credentials_get_expiration(@native)) + exp = Aws::Crt::Native.credentials_get_expiration(@native) + return if exp == UINT64_MAX + + Time.at(exp) end # @return [Credentials] diff --git a/gems/aws-crt-auth/spec/credentials_spec.rb b/gems/aws-crt-auth/spec/credentials_spec.rb index b057924..56c7f88 100644 --- a/gems/aws-crt-auth/spec/credentials_spec.rb +++ b/gems/aws-crt-auth/spec/credentials_spec.rb @@ -24,9 +24,9 @@ module Auth #:nodoc: expect(Credentials.new('akid', 'secret').session_token).to be nil end - it 'defaults the expiration to UINT64_MAX' do - expect(Credentials.new('akid', 'secret').expiration.to_i) - .to eq UINT64_MAX + it 'defaults the expiration to nil' do + expect(Credentials.new('akid', 'secret').expiration) + .to be_nil end it 'accepts a Time for expiration' do diff --git a/gems/aws-crt/native/src/credentials.c b/gems/aws-crt/native/src/credentials.c index 8fc1c23..8cdbe53 100644 --- a/gems/aws-crt/native/src/credentials.c +++ b/gems/aws-crt/native/src/credentials.c @@ -20,12 +20,12 @@ AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( aws_byte_cursor_from_c_str(secret_access_key), aws_byte_cursor_from_c_str(session_token), expiration_timepoint_seconds); -// return aws_credentials_new_from_string( -// allocator, -// aws_string_new_from_c_str(allocator, access_key_id), -// aws_string_new_from_c_str(allocator, secret_access_key), -// session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), -// expiration_timepoint_seconds); + // return aws_credentials_new_from_string( + // allocator, + // aws_string_new_from_c_str(allocator, access_key_id), + // aws_string_new_from_c_str(allocator, secret_access_key), + // session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), + // expiration_timepoint_seconds); } const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials) { From 63577a5952bed8f45f7b139a436e8d38c9d56e43 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 29 Sep 2020 14:43:59 -0700 Subject: [PATCH 06/17] WIP signing config --- gems/aws-crt-auth/lib/aws-crt-auth.rb | 1 + .../lib/aws-crt-auth/credentials.rb | 18 ++--- .../lib/aws-crt-auth/signing_config.rb | 77 ++++++++++++++++++ gems/aws-crt-auth/spec/signing_config_spec.rb | 81 +++++++++++++++++++ gems/aws-crt/lib/aws-crt/native.rb | 6 ++ gems/aws-crt/native/src/api.h | 10 +++ gems/aws-crt/native/src/credentials.c | 2 +- gems/aws-crt/native/src/signing_config.c | 63 +++++++++++++++ 8 files changed, 245 insertions(+), 13 deletions(-) create mode 100644 gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb create mode 100644 gems/aws-crt-auth/spec/signing_config_spec.rb create mode 100644 gems/aws-crt/native/src/signing_config.c diff --git a/gems/aws-crt-auth/lib/aws-crt-auth.rb b/gems/aws-crt-auth/lib/aws-crt-auth.rb index 33d01cd..95ee442 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth.rb @@ -3,6 +3,7 @@ require 'aws-crt' require_relative 'aws-crt-auth/signer' require_relative 'aws-crt-auth/credentials' +require_relative 'aws-crt-auth/signing_config' module Aws module Crt diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index a155fce..1a27aee 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -4,12 +4,14 @@ module Aws module Crt module Auth # Utility class for Credentials. + # @api private class Credentials UINT64_MAX = 18_446_744_073_709_551_615 # @param [String] access_key_id # @param [String] secret_access_key # @param [String] session_token (nil) + # @param [Time|int] expiration (nil) def initialize(access_key_id, secret_access_key, session_token = nil, expiration = nil) if !access_key_id || access_key_id.empty? @@ -20,29 +22,21 @@ def initialize(access_key_id, secret_access_key, raise ArgumentError, 'secret_access_key must be set' end - # access_key_id = FFI::MemoryPointer.from_string(access_key_id) - # access_key_id.autorelease = false - # secret_access_key = FFI::MemoryPointer.from_string(secret_access_key) - # secret_access_key.autorelease = false - # if session_token - # session_token = FFI::MemoryPointer.from_string(session_token) - # session_token.autorelease = false - # end - - native = Aws::Crt.call do - # -1 will give UINT max Aws::Crt::Native.credentials_new( access_key_id, secret_access_key, session_token, - expiration&.to_i || -1 + expiration&.to_i || UINT64_MAX ) end @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) end + # @return [FFI:Pointer] + attr_reader :native + # @return [String, nil] def access_key_id Aws::Crt::Native.credentials_get_access_key_id(@native) if @native diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb b/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb new file mode 100644 index 0000000..d88e293 --- /dev/null +++ b/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module Aws + module Crt + module Auth + # Signing Config + # @api private + class SigningConfig + # @param [Hash] options + # @option options [required, Aws::Crt::Native::signing_algorithm] + # :algorithm + # @option options [required, Aws::Crt::Native::signature_type] + # :signature_type + # @option options [required, Credentials] :credentials + # @option options [required, String] :region + # @option options [required, String] :service + # @option options [Time] :date (Time.now) + # @option options [Array|Proc(String->Boolean)] + # :unsigned_headers ([]) + # @option options [Boolean] :uri_escape_path (true) + # @option options [Boolean] :apply_checksum_header (true) + def initialize(options = {}) + # validation of parameters is handled in signing_config_new + + # create a callback function for aws_should_sign_header_fn + sign_header_fn = extract_unsigned_header_fn( + options[:unsigned_headers] + ) + puts "TODO: imp #{sign_header_fn}" if sign_header_fn + + native = Aws::Crt.call do + Aws::Crt::Native.signing_config_new( + options[:algorithm], + options[:signature_type], + options[:region], + options[:service], + extract_date(options), + options[:credentials]&.native + ) + end + + @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) + end + + # Immediately release this instance's attachment to the underlying + # resources, without waiting for the garbage collector. + # Note that underlying resources will remain alive until nothing + # else is using them. + def release + return unless @native + + @native.free + @native = nil + end + + def self.on_release(native) + Aws::Crt::Native.signing_config_release(native) + end + + private + + def extract_date(options) + (options[:date] || Time.now).to_i + end + + def extract_unsigned_header_fn(unsigned_headers) + if options[:unsigned_headers] && + !options[:unsigned_headers].respond_to?(:call) + unsigned_headers = Set.new(options[:unsigned_headers]) + sign_header_fn = proc { |param| unsigned_headers.include? param } + end + sign_header_fn + end + end + end + end +end diff --git a/gems/aws-crt-auth/spec/signing_config_spec.rb b/gems/aws-crt-auth/spec/signing_config_spec.rb new file mode 100644 index 0000000..d2e982f --- /dev/null +++ b/gems/aws-crt-auth/spec/signing_config_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'weakref' + +module Aws + module Crt + module Auth #:nodoc: + InvalidConfigError = Aws::Crt::Errors.error_class( + 'AWS_AUTH_SIGNING_INVALID_CONFIGURATION' + ) + describe SigningConfig do + describe '#initialize' do + it 'constructs the object' do + creds = Credentials.new('akid', 'secret') + expect do + SigningConfig.new( + signature_type: :http_request_headers, + region: 'us-west-2', + service: 's3', + credentials: creds + ) + end.not_to raise_error + end + + it 'raises an InvalidConfigError when missing a required parameter' do + expect do + SigningConfig.new( + algorithm: :v4, + signature_type: :http_request_headers, + region: 'us-west-2', + service: 's3', + credentials: nil + ) + end.to raise_error(InvalidConfigError) + end + end + + describe '.on_release' do + # Note: Cannot use let with GC tests + it 'cleans up with release' do + creds = Credentials.new('akid', 'secret') + config = SigningConfig.new( + algorithm: :v4, + signature_type: :http_request_headers, + region: 'us-west-2', + service: 's3', + credentials: creds + ) + + expect(config).to_not be_nil + + config.release + check_for_clean_shutdown + end + + if garbage_collect_is_immediate? + it 'cleans up with GC' do + creds = Credentials.new('akid', 'secret') + config = SigningConfig.new( + algorithm: :v4, + signature_type: :http_request_headers, + region: 'us-west-2', + service: 's3', + credentials: creds + ) + weakref = WeakRef.new(config) + expect(weakref.weakref_alive?).to be true + + # force cleanup via GC + config = nil # rubocop:disable Lint/UselessAssignment + ObjectSpace.garbage_collect + expect(weakref.weakref_alive?).to be_falsey + check_for_clean_shutdown + end + end + end + end + end + end +end diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index 4718102..8f60415 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -31,6 +31,12 @@ module Native attach_function :credentials_get_session_token, :aws_crt_credentials_get_session_token, [:pointer], :string attach_function :credentials_get_expiration, :aws_crt_credentials_get_expiration_timepoint_seconds, [:pointer], :uint64 + enum :signing_algorithm, [:v4] + enum :signature_type, %i[http_request_headers http_request_query_params + http_request_chunk http_request_event] + attach_function :signing_config_new, :aws_crt_signing_config_new, %i[signing_algorithm signature_type string string uint64 pointer], :pointer + attach_function :signing_config_release, :aws_crt_signing_config_release, [:pointer], :void + # Internal testing API attach_function :test_error, :aws_crt_test_error, [:int], :int attach_function :test_pointer_error, :aws_crt_test_pointer_error, [], :pointer diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index add4bbd..e33468d 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -54,6 +54,16 @@ AWS_CRT_API const char *aws_crt_credentials_get_secret_access_key(struct aws_cre AWS_CRT_API const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials); AWS_CRT_API uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials); +AWS_CRT_API struct aws_crt_signing_config *aws_crt_signing_config_new( + int algorithm, + int signature_type, + char *region, + char *service, + uint64_t date_epoch_ms, + struct aws_credentials *credentials +); +AWS_CRT_API void aws_crt_signing_config_release(struct aws_crt_signing_config *config); + AWS_EXTERN_C_END #endif /* AWS_CRT_API_H */ diff --git a/gems/aws-crt/native/src/credentials.c b/gems/aws-crt/native/src/credentials.c index 8cdbe53..c41ce57 100644 --- a/gems/aws-crt/native/src/credentials.c +++ b/gems/aws-crt/native/src/credentials.c @@ -7,7 +7,7 @@ #include #include -AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( +struct aws_credentials *aws_crt_credentials_new( const char *access_key_id, const char *secret_access_key, const char *session_token, diff --git a/gems/aws-crt/native/src/signing_config.c b/gems/aws-crt/native/src/signing_config.c new file mode 100644 index 0000000..556641b --- /dev/null +++ b/gems/aws-crt/native/src/signing_config.c @@ -0,0 +1,63 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include "crt.h" + +#include +#include +#include + +struct aws_crt_signing_config { + struct aws_signing_config_aws native; + struct aws_allocator *allocator; + struct aws_atomic_var ref_count; +}; + +struct aws_crt_signing_config *aws_crt_signing_config_new( + int algorithm, + int signature_type, + char *region, + char *service, + uint64_t date_epoch_ms, + struct aws_credentials *credentials +) { + struct aws_allocator *allocator = aws_crt_allocator(); + struct aws_crt_signing_config *config = aws_mem_acquire(allocator, sizeof(struct aws_crt_signing_config)); + if (config == NULL) { + return NULL; + } + config->allocator = allocator; + aws_atomic_init_int(&config->ref_count, 1); + + AWS_ZERO_STRUCT(*config); + config->native.config_type = AWS_SIGNING_CONFIG_AWS; + config->native.algorithm = algorithm; + config->native.signature_type = signature_type; + config->native.region = aws_byte_cursor_from_c_str(region); //TODO: these are not getting copied. Need to use string? + config->native.service = aws_byte_cursor_from_c_str(service); + aws_date_time_init_epoch_millis(&config->native.date, date_epoch_ms); //TODO: should this be ms or sec? + config->native.credentials = credentials; + + //TODO: + // should_sign_header + should_sign_header_ud (set to self?) + // flags for: use_double_uri_encode, should_normalize_uri_path, ect + + if (aws_validate_aws_signing_config_aws(&config->native) != 0) { + aws_crt_signing_config_release(config); + return NULL; + } + + return config; +} + +void aws_crt_signing_config_release(struct aws_crt_signing_config *config) { + if (config == NULL) { + return; + } + + size_t old_value = aws_atomic_fetch_sub(&config->ref_count, 1); + if (old_value == 1) { + aws_mem_release(config->allocator, config); + } +} From ac50adaf6aebccc1b147d6f237e9264b1b4c314d Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 29 Sep 2020 15:12:26 -0700 Subject: [PATCH 07/17] Ensure strings are copied and cleaned up --- .../lib/aws-crt-auth/signing_config.rb | 9 +++++---- gems/aws-crt-auth/spec/signing_config_spec.rb | 1 + gems/aws-crt/native/src/signing_config.c | 17 +++++++++++++++-- 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb b/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb index d88e293..4c687d0 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb @@ -28,6 +28,8 @@ def initialize(options = {}) ) puts "TODO: imp #{sign_header_fn}" if sign_header_fn + # ensure we retain a reference to the credentials to avoid GC + @credentials = options[:credentials] native = Aws::Crt.call do Aws::Crt::Native.signing_config_new( options[:algorithm], @@ -35,7 +37,7 @@ def initialize(options = {}) options[:region], options[:service], extract_date(options), - options[:credentials]&.native + @credentials&.native ) end @@ -64,9 +66,8 @@ def extract_date(options) end def extract_unsigned_header_fn(unsigned_headers) - if options[:unsigned_headers] && - !options[:unsigned_headers].respond_to?(:call) - unsigned_headers = Set.new(options[:unsigned_headers]) + if unsigned_headers && !unsigned_headers.respond_to?(:call) + unsigned_headers = Set.new(unsigned_headers) sign_header_fn = proc { |param| unsigned_headers.include? param } end sign_header_fn diff --git a/gems/aws-crt-auth/spec/signing_config_spec.rb b/gems/aws-crt-auth/spec/signing_config_spec.rb index d2e982f..6eff97d 100644 --- a/gems/aws-crt-auth/spec/signing_config_spec.rb +++ b/gems/aws-crt-auth/spec/signing_config_spec.rb @@ -15,6 +15,7 @@ module Auth #:nodoc: creds = Credentials.new('akid', 'secret') expect do SigningConfig.new( + algorithm: :v4, signature_type: :http_request_headers, region: 'us-west-2', service: 's3', diff --git a/gems/aws-crt/native/src/signing_config.c b/gems/aws-crt/native/src/signing_config.c index 556641b..c1931ec 100644 --- a/gems/aws-crt/native/src/signing_config.c +++ b/gems/aws-crt/native/src/signing_config.c @@ -12,6 +12,8 @@ struct aws_crt_signing_config { struct aws_signing_config_aws native; struct aws_allocator *allocator; struct aws_atomic_var ref_count; + struct aws_string *region_str; + struct aws_string *service_str; }; struct aws_crt_signing_config *aws_crt_signing_config_new( @@ -31,11 +33,16 @@ struct aws_crt_signing_config *aws_crt_signing_config_new( aws_atomic_init_int(&config->ref_count, 1); AWS_ZERO_STRUCT(*config); + //copy string data + config->region_str = aws_string_new_from_c_str(allocator, region); + config->service_str = aws_string_new_from_c_str(allocator, service); + + config->native.config_type = AWS_SIGNING_CONFIG_AWS; config->native.algorithm = algorithm; config->native.signature_type = signature_type; - config->native.region = aws_byte_cursor_from_c_str(region); //TODO: these are not getting copied. Need to use string? - config->native.service = aws_byte_cursor_from_c_str(service); + config->native.region = aws_byte_cursor_from_string(config->region_str); + config->native.service = aws_byte_cursor_from_string(config->service_str); aws_date_time_init_epoch_millis(&config->native.date, date_epoch_ms); //TODO: should this be ms or sec? config->native.credentials = credentials; @@ -58,6 +65,12 @@ void aws_crt_signing_config_release(struct aws_crt_signing_config *config) { size_t old_value = aws_atomic_fetch_sub(&config->ref_count, 1); if (old_value == 1) { + if (config->region_str != NULL) { + aws_string_destroy(config->region_str); + } + if (config->service_str != NULL) { + aws_string_destroy(config->service_str); + } aws_mem_release(config->allocator, config); } } From 110459c5856cee834ff711e0d035569fd23ae19f Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Tue, 29 Sep 2020 15:28:49 -0700 Subject: [PATCH 08/17] Clang cleanup --- gems/aws-crt/native/src/api.h | 3 +-- gems/aws-crt/native/src/signing_config.c | 11 +++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index e33468d..5a0a9a0 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -60,8 +60,7 @@ AWS_CRT_API struct aws_crt_signing_config *aws_crt_signing_config_new( char *region, char *service, uint64_t date_epoch_ms, - struct aws_credentials *credentials -); + struct aws_credentials *credentials); AWS_CRT_API void aws_crt_signing_config_release(struct aws_crt_signing_config *config); AWS_EXTERN_C_END diff --git a/gems/aws-crt/native/src/signing_config.c b/gems/aws-crt/native/src/signing_config.c index c1931ec..a8d8a13 100644 --- a/gems/aws-crt/native/src/signing_config.c +++ b/gems/aws-crt/native/src/signing_config.c @@ -4,8 +4,8 @@ */ #include "crt.h" -#include #include +#include #include struct aws_crt_signing_config { @@ -22,8 +22,7 @@ struct aws_crt_signing_config *aws_crt_signing_config_new( char *region, char *service, uint64_t date_epoch_ms, - struct aws_credentials *credentials -) { + struct aws_credentials *credentials) { struct aws_allocator *allocator = aws_crt_allocator(); struct aws_crt_signing_config *config = aws_mem_acquire(allocator, sizeof(struct aws_crt_signing_config)); if (config == NULL) { @@ -33,7 +32,7 @@ struct aws_crt_signing_config *aws_crt_signing_config_new( aws_atomic_init_int(&config->ref_count, 1); AWS_ZERO_STRUCT(*config); - //copy string data + // copy string data config->region_str = aws_string_new_from_c_str(allocator, region); config->service_str = aws_string_new_from_c_str(allocator, service); @@ -43,10 +42,10 @@ struct aws_crt_signing_config *aws_crt_signing_config_new( config->native.signature_type = signature_type; config->native.region = aws_byte_cursor_from_string(config->region_str); config->native.service = aws_byte_cursor_from_string(config->service_str); - aws_date_time_init_epoch_millis(&config->native.date, date_epoch_ms); //TODO: should this be ms or sec? + aws_date_time_init_epoch_millis(&config->native.date, date_epoch_ms); // TODO: should this be ms or sec? config->native.credentials = credentials; - //TODO: + // TODO: // should_sign_header + should_sign_header_ud (set to self?) // flags for: use_double_uri_encode, should_normalize_uri_path, ect From 3e8c886e1fd1cb6ee768cc244b0a8fbacb46e5eb Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 30 Sep 2020 10:56:56 -0700 Subject: [PATCH 09/17] WIP - Signable impl --- format-check.sh | 2 +- gems/aws-crt-auth/lib/aws-crt-auth.rb | 3 +- .../aws-crt-auth/lib/aws-crt-auth/signable.rb | 55 ++++ gems/aws-crt-auth/spec/signable_spec.rb | 21 ++ gems/aws-crt/lib/aws-crt/native.rb | 6 + gems/aws-crt/native/src/api.h | 5 + gems/aws-crt/native/src/signable.c | 241 ++++++++++++++++++ 7 files changed, 331 insertions(+), 2 deletions(-) create mode 100644 gems/aws-crt-auth/lib/aws-crt-auth/signable.rb create mode 100644 gems/aws-crt-auth/spec/signable_spec.rb create mode 100644 gems/aws-crt/native/src/signable.c diff --git a/format-check.sh b/format-check.sh index 8429423..9d19e14 100755 --- a/format-check.sh +++ b/format-check.sh @@ -10,7 +10,7 @@ if NOT type $CLANG_FORMAT 2> /dev/null ; then fi FAIL=0 -SOURCE_FILES=`find aws-crt/native/src -type f \( -name '*.h' -o -name '*.c' \)` +SOURCE_FILES=`find gems/aws-crt/native/src -type f \( -name '*.h' -o -name '*.c' \)` for i in $SOURCE_FILES do $CLANG_FORMAT -output-replacements-xml $i | grep -c " /dev/null diff --git a/gems/aws-crt-auth/lib/aws-crt-auth.rb b/gems/aws-crt-auth/lib/aws-crt-auth.rb index 95ee442..6455ae2 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth.rb @@ -1,9 +1,10 @@ # frozen_string_literal: true require 'aws-crt' -require_relative 'aws-crt-auth/signer' require_relative 'aws-crt-auth/credentials' require_relative 'aws-crt-auth/signing_config' +require_relative 'aws-crt-auth/signable' +require_relative 'aws-crt-auth/signer' module Aws module Crt diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/signable.rb b/gems/aws-crt-auth/lib/aws-crt-auth/signable.rb new file mode 100644 index 0000000..d3499f9 --- /dev/null +++ b/gems/aws-crt-auth/lib/aws-crt-auth/signable.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Aws + module Crt + module Auth + # Signing Config + # @api private + class Signable + # @param [Hash] options + # @option options [required, Aws::Crt::Native::signing_algorithm] + # :algorithm + def initialize(options = {}) + # validation of parameters is handled in signing_config_new + + # create a callback function for aws_should_sign_header_fn + sign_header_fn = extract_unsigned_header_fn( + options[:unsigned_headers] + ) + puts "TODO: imp #{sign_header_fn}" if sign_header_fn + + # ensure we retain a reference to the credentials to avoid GC + @credentials = options[:credentials] + native = Aws::Crt.call do + Aws::Crt::Native.signing_config_new( + options[:algorithm], + options[:signature_type], + options[:region], + options[:service], + extract_date(options), + @credentials&.native + ) + end + + @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) + end + + # Immediately release this instance's attachment to the underlying + # resources, without waiting for the garbage collector. + # Note that underlying resources will remain alive until nothing + # else is using them. + def release + return unless @native + + @native.free + @native = nil + end + + def self.on_release(native) + Aws::Crt::Native.signing_config_release(native) + end + + end + end + end +end diff --git a/gems/aws-crt-auth/spec/signable_spec.rb b/gems/aws-crt-auth/spec/signable_spec.rb new file mode 100644 index 0000000..34562e6 --- /dev/null +++ b/gems/aws-crt-auth/spec/signable_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' +require 'weakref' + +module Aws + module Crt + module Auth #:nodoc: + describe Signable do + + it 'works' do + ptr = Aws::Crt::Native::signable_new() + puts ptr + puts Aws::Crt::Native::signable_set_property(ptr, 'name', 'value') + puts Aws::Crt::Native::signable_get_property(ptr, 'name') + Aws::Crt::Native::signable_release(ptr) + end + end + end + end +end \ No newline at end of file diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index 8f60415..050a8ca 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -37,6 +37,12 @@ module Native attach_function :signing_config_new, :aws_crt_signing_config_new, %i[signing_algorithm signature_type string string uint64 pointer], :pointer attach_function :signing_config_release, :aws_crt_signing_config_release, [:pointer], :void + attach_function :signable_new, :aws_crt_signable_new, [], :pointer + attach_function :signable_release, :aws_crt_signable_release, [:pointer], :void + attach_function :signable_set_property, :aws_crt_signable_set_property, [:pointer, :string, :string], :int + attach_function :signable_get_property, :aws_crt_signable_get_property, [:pointer, :string], :string + + # Internal testing API attach_function :test_error, :aws_crt_test_error, [:int], :int attach_function :test_pointer_error, :aws_crt_test_pointer_error, [], :pointer diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index 5a0a9a0..c2f204c 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -63,6 +63,11 @@ AWS_CRT_API struct aws_crt_signing_config *aws_crt_signing_config_new( struct aws_credentials *credentials); AWS_CRT_API void aws_crt_signing_config_release(struct aws_crt_signing_config *config); +AWS_CRT_API struct aws_signable *aws_crt_signable_new(void); +AWS_CRT_API void aws_crt_signable_release(struct aws_signable *signable); +AWS_CRT_API int aws_crt_signable_set_property(struct aws_signable *signable, const char *property_name, const char *property_value); +AWS_CRT_API const char *aws_crt_signable_get_property(struct aws_signable *signable, const char *property_name); + AWS_EXTERN_C_END #endif /* AWS_CRT_API_H */ diff --git a/gems/aws-crt/native/src/signable.c b/gems/aws-crt/native/src/signable.c new file mode 100644 index 0000000..d44ff3f --- /dev/null +++ b/gems/aws-crt/native/src/signable.c @@ -0,0 +1,241 @@ +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +#include "crt.h" + +#include +#include +#include + +#define INITIAL_AWS_CRT_SIGNABLE_PROPERTIES_SIZE 10 +#define INITIAL_AWS_CRT_SIGNABLE_PROPERTY_LISTS_TABLE_SIZE 10 +#define INITIAL_AWS_CRT_SIGNABLE_PROPERTY_LIST_SIZE 10 + +struct aws_crt_signable_property { + struct aws_string *name; + struct aws_string *value; +}; + +struct aws_crt_signable_impl { + struct aws_allocator *allocator; + struct aws_hash_table properties; + struct aws_hash_table property_lists; +}; + +static int s_aws_crt_signable_get_property( + const struct aws_signable *signable, + const struct aws_string *name, + struct aws_byte_cursor *out_value) { + + struct aws_crt_signable_impl *impl = signable->impl; + + AWS_ZERO_STRUCT(*out_value); + + struct aws_hash_element *element = NULL; + aws_hash_table_find(&impl->properties, name, &element); + + if (element != NULL) { + *out_value = aws_byte_cursor_from_string(element->value); + return AWS_OP_SUCCESS; + } + + return aws_raise_error(AWS_ERROR_HASHTBL_ITEM_NOT_FOUND); +} + +static int s_aws_crt_signable_get_property_list( + const struct aws_signable *signable, + const struct aws_string *name, + struct aws_array_list **out_list) { + + (void)signable; + (void)name; + (void)out_list; + +// struct aws_crt_signable_impl *impl = signable->impl; +// +// *out_list = NULL; +// +// if (aws_string_eq(name, g_aws_http_headers_property_list_name)) { +// *out_list = &impl->headers; +// } else { +// return AWS_OP_ERR; +// } + + return AWS_OP_SUCCESS; +} + +static int s_aws_crt_signable_get_payload_stream( + const struct aws_signable *signable, + struct aws_input_stream **out_input_stream) { + +// struct aws_crt_signable_impl *impl = signable->impl; +// *out_input_stream = aws_http_message_get_body_stream(impl->request); + (void)signable; + (void)out_input_stream; + + return AWS_OP_SUCCESS; +} + +static void s_aws_crt_signable_destroy(struct aws_signable *signable) { + if (signable == NULL) { + return; + } + + struct aws_crt_signable_impl *impl = signable->impl; + if (impl == NULL) { + return; + } + +// aws_array_list_clean_up(&impl->headers); +// aws_mem_release(signable->allocator, signable); +} + +static struct aws_signable_vtable s_aws_crt_signable_vtable = { + .get_property = s_aws_crt_signable_get_property, + .get_property_list = s_aws_crt_signable_get_property_list, + .get_payload_stream = s_aws_crt_signable_get_payload_stream, + .destroy = s_aws_crt_signable_destroy, +}; + +static void s_aws_crt_signable_property_clean_up(struct aws_crt_signable_property *pair) { + aws_string_destroy(pair->name); + aws_string_destroy(pair->value); +} + +static void s_aws_hash_callback_property_list_destroy(void *value) { + struct aws_array_list *property_list = value; + + size_t property_count = aws_array_list_length(property_list); + for (size_t i = 0; i < property_count; ++i) { + struct aws_crt_signable_property property; + AWS_ZERO_STRUCT(property); + + if (aws_array_list_get_at(property_list, &property, i)) { + continue; + } + + s_aws_crt_signable_property_clean_up(&property); + } + + struct aws_allocator *allocator = property_list->alloc; + aws_array_list_clean_up(property_list); + + aws_mem_release(allocator, property_list); +} + +struct aws_signable *aws_crt_signable_new(void) { + struct aws_allocator *allocator = aws_crt_allocator(); + + struct aws_signable *signable = NULL; + struct aws_crt_signable_impl *impl = NULL; + aws_mem_acquire_many( + allocator, 2, &signable, sizeof(struct aws_signable), &impl, sizeof(struct aws_crt_signable_impl)); + + if (signable == NULL || impl == NULL) { + return NULL; + } + + AWS_ZERO_STRUCT(*signable); + AWS_ZERO_STRUCT(*impl); + + signable->allocator = allocator; + signable->vtable = &s_aws_crt_signable_vtable; + signable->impl = impl; + + impl->allocator = allocator; + if (aws_hash_table_init( + &impl->properties, + allocator, + INITIAL_AWS_CRT_SIGNABLE_PROPERTIES_SIZE, + aws_hash_string, + aws_hash_callback_string_eq, + aws_hash_callback_string_destroy, + aws_hash_callback_string_destroy) || + aws_hash_table_init( + &impl->property_lists, + allocator, + INITIAL_AWS_CRT_SIGNABLE_PROPERTY_LISTS_TABLE_SIZE, + aws_hash_string, + aws_hash_callback_string_eq, + aws_hash_callback_string_destroy, + s_aws_hash_callback_property_list_destroy)) { + goto on_error; + } + + return signable; + + on_error: + + aws_signable_destroy(signable); + + return NULL; +} + +int aws_crt_signable_set_property( + struct aws_signable *signable, + const char *property_name, + const char *property_value) { + + if (signable == NULL) { + return AWS_OP_ERR; + } + + struct aws_crt_signable_impl *impl = signable->impl; + if (impl == NULL) { + return AWS_OP_ERR; + } + + struct aws_string *name = NULL; + struct aws_string *value = NULL; + + name = aws_string_new_from_c_str(impl->allocator, property_name); + value = aws_string_new_from_c_str(impl->allocator, property_value); + if (name == NULL || value == NULL) { + goto on_error; + } + + if (aws_hash_table_put(&impl->properties, name, value, NULL)) { + goto on_error; + } + + return AWS_OP_SUCCESS; + +on_error: + + aws_string_destroy(name); + aws_string_destroy(value); + + return AWS_OP_ERR; +} + +const char *aws_crt_signable_get_property(struct aws_signable *signable, const char *property_name) { + AWS_PRECONDITION(signable); + struct aws_crt_signable_impl *impl = signable->impl; + AWS_PRECONDITION(impl); + + struct aws_string *name = NULL; + struct aws_byte_cursor out_value; + + name = aws_string_new_from_c_str(impl->allocator, property_name); + if (name == NULL) { + goto on_error; + } + + int success = s_aws_crt_signable_get_property(signable, name, &out_value); + if (success != 0) { + goto on_error; + } + + return (char*)out_value.ptr; + + on_error: + + aws_string_destroy(name); + // TODO: Does out_value need to be cleaned up? + return NULL; +} + +void aws_crt_signable_release(struct aws_signable *signable) { + aws_signable_destroy(signable); +} From 0cbdd1829ae99609b061357a199df0e30ef8a846 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 30 Sep 2020 13:54:31 -0700 Subject: [PATCH 10/17] Clean up PR - only include credentials binding --- gems/aws-crt-auth/lib/aws-crt-auth.rb | 2 - .../aws-crt-auth/lib/aws-crt-auth/signable.rb | 55 ---- .../lib/aws-crt-auth/signing_config.rb | 78 ------ gems/aws-crt-auth/spec/signable_spec.rb | 21 -- gems/aws-crt-auth/spec/signing_config_spec.rb | 82 ------ gems/aws-crt/lib/aws-crt/native.rb | 12 - gems/aws-crt/native/src/api.h | 14 - gems/aws-crt/native/src/signable.c | 241 ------------------ gems/aws-crt/native/src/signing_config.c | 75 ------ 9 files changed, 580 deletions(-) delete mode 100644 gems/aws-crt-auth/lib/aws-crt-auth/signable.rb delete mode 100644 gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb delete mode 100644 gems/aws-crt-auth/spec/signable_spec.rb delete mode 100644 gems/aws-crt-auth/spec/signing_config_spec.rb delete mode 100644 gems/aws-crt/native/src/signable.c delete mode 100644 gems/aws-crt/native/src/signing_config.c diff --git a/gems/aws-crt-auth/lib/aws-crt-auth.rb b/gems/aws-crt-auth/lib/aws-crt-auth.rb index 6455ae2..736774e 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth.rb @@ -2,8 +2,6 @@ require 'aws-crt' require_relative 'aws-crt-auth/credentials' -require_relative 'aws-crt-auth/signing_config' -require_relative 'aws-crt-auth/signable' require_relative 'aws-crt-auth/signer' module Aws diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/signable.rb b/gems/aws-crt-auth/lib/aws-crt-auth/signable.rb deleted file mode 100644 index d3499f9..0000000 --- a/gems/aws-crt-auth/lib/aws-crt-auth/signable.rb +++ /dev/null @@ -1,55 +0,0 @@ -# frozen_string_literal: true - -module Aws - module Crt - module Auth - # Signing Config - # @api private - class Signable - # @param [Hash] options - # @option options [required, Aws::Crt::Native::signing_algorithm] - # :algorithm - def initialize(options = {}) - # validation of parameters is handled in signing_config_new - - # create a callback function for aws_should_sign_header_fn - sign_header_fn = extract_unsigned_header_fn( - options[:unsigned_headers] - ) - puts "TODO: imp #{sign_header_fn}" if sign_header_fn - - # ensure we retain a reference to the credentials to avoid GC - @credentials = options[:credentials] - native = Aws::Crt.call do - Aws::Crt::Native.signing_config_new( - options[:algorithm], - options[:signature_type], - options[:region], - options[:service], - extract_date(options), - @credentials&.native - ) - end - - @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) - end - - # Immediately release this instance's attachment to the underlying - # resources, without waiting for the garbage collector. - # Note that underlying resources will remain alive until nothing - # else is using them. - def release - return unless @native - - @native.free - @native = nil - end - - def self.on_release(native) - Aws::Crt::Native.signing_config_release(native) - end - - end - end - end -end diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb b/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb deleted file mode 100644 index 4c687d0..0000000 --- a/gems/aws-crt-auth/lib/aws-crt-auth/signing_config.rb +++ /dev/null @@ -1,78 +0,0 @@ -# frozen_string_literal: true - -module Aws - module Crt - module Auth - # Signing Config - # @api private - class SigningConfig - # @param [Hash] options - # @option options [required, Aws::Crt::Native::signing_algorithm] - # :algorithm - # @option options [required, Aws::Crt::Native::signature_type] - # :signature_type - # @option options [required, Credentials] :credentials - # @option options [required, String] :region - # @option options [required, String] :service - # @option options [Time] :date (Time.now) - # @option options [Array|Proc(String->Boolean)] - # :unsigned_headers ([]) - # @option options [Boolean] :uri_escape_path (true) - # @option options [Boolean] :apply_checksum_header (true) - def initialize(options = {}) - # validation of parameters is handled in signing_config_new - - # create a callback function for aws_should_sign_header_fn - sign_header_fn = extract_unsigned_header_fn( - options[:unsigned_headers] - ) - puts "TODO: imp #{sign_header_fn}" if sign_header_fn - - # ensure we retain a reference to the credentials to avoid GC - @credentials = options[:credentials] - native = Aws::Crt.call do - Aws::Crt::Native.signing_config_new( - options[:algorithm], - options[:signature_type], - options[:region], - options[:service], - extract_date(options), - @credentials&.native - ) - end - - @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) - end - - # Immediately release this instance's attachment to the underlying - # resources, without waiting for the garbage collector. - # Note that underlying resources will remain alive until nothing - # else is using them. - def release - return unless @native - - @native.free - @native = nil - end - - def self.on_release(native) - Aws::Crt::Native.signing_config_release(native) - end - - private - - def extract_date(options) - (options[:date] || Time.now).to_i - end - - def extract_unsigned_header_fn(unsigned_headers) - if unsigned_headers && !unsigned_headers.respond_to?(:call) - unsigned_headers = Set.new(unsigned_headers) - sign_header_fn = proc { |param| unsigned_headers.include? param } - end - sign_header_fn - end - end - end - end -end diff --git a/gems/aws-crt-auth/spec/signable_spec.rb b/gems/aws-crt-auth/spec/signable_spec.rb deleted file mode 100644 index 34562e6..0000000 --- a/gems/aws-crt-auth/spec/signable_spec.rb +++ /dev/null @@ -1,21 +0,0 @@ -# frozen_string_literal: true - -require_relative 'spec_helper' -require 'weakref' - -module Aws - module Crt - module Auth #:nodoc: - describe Signable do - - it 'works' do - ptr = Aws::Crt::Native::signable_new() - puts ptr - puts Aws::Crt::Native::signable_set_property(ptr, 'name', 'value') - puts Aws::Crt::Native::signable_get_property(ptr, 'name') - Aws::Crt::Native::signable_release(ptr) - end - end - end - end -end \ No newline at end of file diff --git a/gems/aws-crt-auth/spec/signing_config_spec.rb b/gems/aws-crt-auth/spec/signing_config_spec.rb deleted file mode 100644 index 6eff97d..0000000 --- a/gems/aws-crt-auth/spec/signing_config_spec.rb +++ /dev/null @@ -1,82 +0,0 @@ -# frozen_string_literal: true - -require_relative 'spec_helper' -require 'weakref' - -module Aws - module Crt - module Auth #:nodoc: - InvalidConfigError = Aws::Crt::Errors.error_class( - 'AWS_AUTH_SIGNING_INVALID_CONFIGURATION' - ) - describe SigningConfig do - describe '#initialize' do - it 'constructs the object' do - creds = Credentials.new('akid', 'secret') - expect do - SigningConfig.new( - algorithm: :v4, - signature_type: :http_request_headers, - region: 'us-west-2', - service: 's3', - credentials: creds - ) - end.not_to raise_error - end - - it 'raises an InvalidConfigError when missing a required parameter' do - expect do - SigningConfig.new( - algorithm: :v4, - signature_type: :http_request_headers, - region: 'us-west-2', - service: 's3', - credentials: nil - ) - end.to raise_error(InvalidConfigError) - end - end - - describe '.on_release' do - # Note: Cannot use let with GC tests - it 'cleans up with release' do - creds = Credentials.new('akid', 'secret') - config = SigningConfig.new( - algorithm: :v4, - signature_type: :http_request_headers, - region: 'us-west-2', - service: 's3', - credentials: creds - ) - - expect(config).to_not be_nil - - config.release - check_for_clean_shutdown - end - - if garbage_collect_is_immediate? - it 'cleans up with GC' do - creds = Credentials.new('akid', 'secret') - config = SigningConfig.new( - algorithm: :v4, - signature_type: :http_request_headers, - region: 'us-west-2', - service: 's3', - credentials: creds - ) - weakref = WeakRef.new(config) - expect(weakref.weakref_alive?).to be true - - # force cleanup via GC - config = nil # rubocop:disable Lint/UselessAssignment - ObjectSpace.garbage_collect - expect(weakref.weakref_alive?).to be_falsey - check_for_clean_shutdown - end - end - end - end - end - end -end diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index 050a8ca..4718102 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -31,18 +31,6 @@ module Native attach_function :credentials_get_session_token, :aws_crt_credentials_get_session_token, [:pointer], :string attach_function :credentials_get_expiration, :aws_crt_credentials_get_expiration_timepoint_seconds, [:pointer], :uint64 - enum :signing_algorithm, [:v4] - enum :signature_type, %i[http_request_headers http_request_query_params - http_request_chunk http_request_event] - attach_function :signing_config_new, :aws_crt_signing_config_new, %i[signing_algorithm signature_type string string uint64 pointer], :pointer - attach_function :signing_config_release, :aws_crt_signing_config_release, [:pointer], :void - - attach_function :signable_new, :aws_crt_signable_new, [], :pointer - attach_function :signable_release, :aws_crt_signable_release, [:pointer], :void - attach_function :signable_set_property, :aws_crt_signable_set_property, [:pointer, :string, :string], :int - attach_function :signable_get_property, :aws_crt_signable_get_property, [:pointer, :string], :string - - # Internal testing API attach_function :test_error, :aws_crt_test_error, [:int], :int attach_function :test_pointer_error, :aws_crt_test_pointer_error, [], :pointer diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index c2f204c..add4bbd 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -54,20 +54,6 @@ AWS_CRT_API const char *aws_crt_credentials_get_secret_access_key(struct aws_cre AWS_CRT_API const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials); AWS_CRT_API uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials); -AWS_CRT_API struct aws_crt_signing_config *aws_crt_signing_config_new( - int algorithm, - int signature_type, - char *region, - char *service, - uint64_t date_epoch_ms, - struct aws_credentials *credentials); -AWS_CRT_API void aws_crt_signing_config_release(struct aws_crt_signing_config *config); - -AWS_CRT_API struct aws_signable *aws_crt_signable_new(void); -AWS_CRT_API void aws_crt_signable_release(struct aws_signable *signable); -AWS_CRT_API int aws_crt_signable_set_property(struct aws_signable *signable, const char *property_name, const char *property_value); -AWS_CRT_API const char *aws_crt_signable_get_property(struct aws_signable *signable, const char *property_name); - AWS_EXTERN_C_END #endif /* AWS_CRT_API_H */ diff --git a/gems/aws-crt/native/src/signable.c b/gems/aws-crt/native/src/signable.c deleted file mode 100644 index d44ff3f..0000000 --- a/gems/aws-crt/native/src/signable.c +++ /dev/null @@ -1,241 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ -#include "crt.h" - -#include -#include -#include - -#define INITIAL_AWS_CRT_SIGNABLE_PROPERTIES_SIZE 10 -#define INITIAL_AWS_CRT_SIGNABLE_PROPERTY_LISTS_TABLE_SIZE 10 -#define INITIAL_AWS_CRT_SIGNABLE_PROPERTY_LIST_SIZE 10 - -struct aws_crt_signable_property { - struct aws_string *name; - struct aws_string *value; -}; - -struct aws_crt_signable_impl { - struct aws_allocator *allocator; - struct aws_hash_table properties; - struct aws_hash_table property_lists; -}; - -static int s_aws_crt_signable_get_property( - const struct aws_signable *signable, - const struct aws_string *name, - struct aws_byte_cursor *out_value) { - - struct aws_crt_signable_impl *impl = signable->impl; - - AWS_ZERO_STRUCT(*out_value); - - struct aws_hash_element *element = NULL; - aws_hash_table_find(&impl->properties, name, &element); - - if (element != NULL) { - *out_value = aws_byte_cursor_from_string(element->value); - return AWS_OP_SUCCESS; - } - - return aws_raise_error(AWS_ERROR_HASHTBL_ITEM_NOT_FOUND); -} - -static int s_aws_crt_signable_get_property_list( - const struct aws_signable *signable, - const struct aws_string *name, - struct aws_array_list **out_list) { - - (void)signable; - (void)name; - (void)out_list; - -// struct aws_crt_signable_impl *impl = signable->impl; -// -// *out_list = NULL; -// -// if (aws_string_eq(name, g_aws_http_headers_property_list_name)) { -// *out_list = &impl->headers; -// } else { -// return AWS_OP_ERR; -// } - - return AWS_OP_SUCCESS; -} - -static int s_aws_crt_signable_get_payload_stream( - const struct aws_signable *signable, - struct aws_input_stream **out_input_stream) { - -// struct aws_crt_signable_impl *impl = signable->impl; -// *out_input_stream = aws_http_message_get_body_stream(impl->request); - (void)signable; - (void)out_input_stream; - - return AWS_OP_SUCCESS; -} - -static void s_aws_crt_signable_destroy(struct aws_signable *signable) { - if (signable == NULL) { - return; - } - - struct aws_crt_signable_impl *impl = signable->impl; - if (impl == NULL) { - return; - } - -// aws_array_list_clean_up(&impl->headers); -// aws_mem_release(signable->allocator, signable); -} - -static struct aws_signable_vtable s_aws_crt_signable_vtable = { - .get_property = s_aws_crt_signable_get_property, - .get_property_list = s_aws_crt_signable_get_property_list, - .get_payload_stream = s_aws_crt_signable_get_payload_stream, - .destroy = s_aws_crt_signable_destroy, -}; - -static void s_aws_crt_signable_property_clean_up(struct aws_crt_signable_property *pair) { - aws_string_destroy(pair->name); - aws_string_destroy(pair->value); -} - -static void s_aws_hash_callback_property_list_destroy(void *value) { - struct aws_array_list *property_list = value; - - size_t property_count = aws_array_list_length(property_list); - for (size_t i = 0; i < property_count; ++i) { - struct aws_crt_signable_property property; - AWS_ZERO_STRUCT(property); - - if (aws_array_list_get_at(property_list, &property, i)) { - continue; - } - - s_aws_crt_signable_property_clean_up(&property); - } - - struct aws_allocator *allocator = property_list->alloc; - aws_array_list_clean_up(property_list); - - aws_mem_release(allocator, property_list); -} - -struct aws_signable *aws_crt_signable_new(void) { - struct aws_allocator *allocator = aws_crt_allocator(); - - struct aws_signable *signable = NULL; - struct aws_crt_signable_impl *impl = NULL; - aws_mem_acquire_many( - allocator, 2, &signable, sizeof(struct aws_signable), &impl, sizeof(struct aws_crt_signable_impl)); - - if (signable == NULL || impl == NULL) { - return NULL; - } - - AWS_ZERO_STRUCT(*signable); - AWS_ZERO_STRUCT(*impl); - - signable->allocator = allocator; - signable->vtable = &s_aws_crt_signable_vtable; - signable->impl = impl; - - impl->allocator = allocator; - if (aws_hash_table_init( - &impl->properties, - allocator, - INITIAL_AWS_CRT_SIGNABLE_PROPERTIES_SIZE, - aws_hash_string, - aws_hash_callback_string_eq, - aws_hash_callback_string_destroy, - aws_hash_callback_string_destroy) || - aws_hash_table_init( - &impl->property_lists, - allocator, - INITIAL_AWS_CRT_SIGNABLE_PROPERTY_LISTS_TABLE_SIZE, - aws_hash_string, - aws_hash_callback_string_eq, - aws_hash_callback_string_destroy, - s_aws_hash_callback_property_list_destroy)) { - goto on_error; - } - - return signable; - - on_error: - - aws_signable_destroy(signable); - - return NULL; -} - -int aws_crt_signable_set_property( - struct aws_signable *signable, - const char *property_name, - const char *property_value) { - - if (signable == NULL) { - return AWS_OP_ERR; - } - - struct aws_crt_signable_impl *impl = signable->impl; - if (impl == NULL) { - return AWS_OP_ERR; - } - - struct aws_string *name = NULL; - struct aws_string *value = NULL; - - name = aws_string_new_from_c_str(impl->allocator, property_name); - value = aws_string_new_from_c_str(impl->allocator, property_value); - if (name == NULL || value == NULL) { - goto on_error; - } - - if (aws_hash_table_put(&impl->properties, name, value, NULL)) { - goto on_error; - } - - return AWS_OP_SUCCESS; - -on_error: - - aws_string_destroy(name); - aws_string_destroy(value); - - return AWS_OP_ERR; -} - -const char *aws_crt_signable_get_property(struct aws_signable *signable, const char *property_name) { - AWS_PRECONDITION(signable); - struct aws_crt_signable_impl *impl = signable->impl; - AWS_PRECONDITION(impl); - - struct aws_string *name = NULL; - struct aws_byte_cursor out_value; - - name = aws_string_new_from_c_str(impl->allocator, property_name); - if (name == NULL) { - goto on_error; - } - - int success = s_aws_crt_signable_get_property(signable, name, &out_value); - if (success != 0) { - goto on_error; - } - - return (char*)out_value.ptr; - - on_error: - - aws_string_destroy(name); - // TODO: Does out_value need to be cleaned up? - return NULL; -} - -void aws_crt_signable_release(struct aws_signable *signable) { - aws_signable_destroy(signable); -} diff --git a/gems/aws-crt/native/src/signing_config.c b/gems/aws-crt/native/src/signing_config.c deleted file mode 100644 index a8d8a13..0000000 --- a/gems/aws-crt/native/src/signing_config.c +++ /dev/null @@ -1,75 +0,0 @@ -/** - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ -#include "crt.h" - -#include -#include -#include - -struct aws_crt_signing_config { - struct aws_signing_config_aws native; - struct aws_allocator *allocator; - struct aws_atomic_var ref_count; - struct aws_string *region_str; - struct aws_string *service_str; -}; - -struct aws_crt_signing_config *aws_crt_signing_config_new( - int algorithm, - int signature_type, - char *region, - char *service, - uint64_t date_epoch_ms, - struct aws_credentials *credentials) { - struct aws_allocator *allocator = aws_crt_allocator(); - struct aws_crt_signing_config *config = aws_mem_acquire(allocator, sizeof(struct aws_crt_signing_config)); - if (config == NULL) { - return NULL; - } - config->allocator = allocator; - aws_atomic_init_int(&config->ref_count, 1); - - AWS_ZERO_STRUCT(*config); - // copy string data - config->region_str = aws_string_new_from_c_str(allocator, region); - config->service_str = aws_string_new_from_c_str(allocator, service); - - - config->native.config_type = AWS_SIGNING_CONFIG_AWS; - config->native.algorithm = algorithm; - config->native.signature_type = signature_type; - config->native.region = aws_byte_cursor_from_string(config->region_str); - config->native.service = aws_byte_cursor_from_string(config->service_str); - aws_date_time_init_epoch_millis(&config->native.date, date_epoch_ms); // TODO: should this be ms or sec? - config->native.credentials = credentials; - - // TODO: - // should_sign_header + should_sign_header_ud (set to self?) - // flags for: use_double_uri_encode, should_normalize_uri_path, ect - - if (aws_validate_aws_signing_config_aws(&config->native) != 0) { - aws_crt_signing_config_release(config); - return NULL; - } - - return config; -} - -void aws_crt_signing_config_release(struct aws_crt_signing_config *config) { - if (config == NULL) { - return; - } - - size_t old_value = aws_atomic_fetch_sub(&config->ref_count, 1); - if (old_value == 1) { - if (config->region_str != NULL) { - aws_string_destroy(config->region_str); - } - if (config->service_str != NULL) { - aws_string_destroy(config->service_str); - } - aws_mem_release(config->allocator, config); - } -} From 86ce4e70c7a833181c17fedce8cef77306dc0856 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 30 Sep 2020 15:13:33 -0700 Subject: [PATCH 11/17] Add a ManagedNative mixin module to add some generic functionality. --- .../lib/aws-crt-auth/credentials.rb | 44 +++------- gems/aws-crt-auth/spec/credentials_spec.rb | 2 - gems/aws-crt/lib/aws-crt.rb | 1 + gems/aws-crt/lib/aws-crt/io.rb | 22 +---- gems/aws-crt/lib/aws-crt/managed_native.rb | 86 +++++++++++++++++++ gems/aws-crt/native/src/api.h | 8 +- gems/aws-crt/native/src/credentials.c | 14 +-- 7 files changed, 110 insertions(+), 67 deletions(-) create mode 100644 gems/aws-crt/lib/aws-crt/managed_native.rb diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index 1a27aee..4870885 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -6,12 +6,16 @@ module Auth # Utility class for Credentials. # @api private class Credentials + include Aws::Crt::ManagedNative + native_destroy Aws::Crt::Native.method(:credentials_release) + UINT64_MAX = 18_446_744_073_709_551_615 # @param [String] access_key_id # @param [String] secret_access_key # @param [String] session_token (nil) - # @param [Time|int] expiration (nil) + # @param [Time|int] expiration (nil) - Either a Time or if an int + # seconds since unix epoch def initialize(access_key_id, secret_access_key, session_token = nil, expiration = nil) if !access_key_id || access_key_id.empty? @@ -22,7 +26,7 @@ def initialize(access_key_id, secret_access_key, raise ArgumentError, 'secret_access_key must be set' end - native = Aws::Crt.call do + manage_native do Aws::Crt::Native.credentials_new( access_key_id, secret_access_key, @@ -30,33 +34,26 @@ def initialize(access_key_id, secret_access_key, expiration&.to_i || UINT64_MAX ) end - - @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) end - # @return [FFI:Pointer] - attr_reader :native - # @return [String, nil] def access_key_id - Aws::Crt::Native.credentials_get_access_key_id(@native) if @native + Aws::Crt::Native.credentials_get_access_key_id(native) end # @return [String, nil] def secret_access_key - Aws::Crt::Native.credentials_get_secret_access_key(@native) if @native + Aws::Crt::Native.credentials_get_secret_access_key(native) end # @return [String, nil] def session_token - Aws::Crt::Native.credentials_get_session_token(@native) if @native + Aws::Crt::Native.credentials_get_session_token(native) end # @return [Time,nil] def expiration - return unless @native - - exp = Aws::Crt::Native.credentials_get_expiration(@native) + exp = Aws::Crt::Native.credentials_get_expiration(native) return if exp == UINT64_MAX Time.at(exp) @@ -70,11 +67,7 @@ def credentials # @return [Boolean] Returns `true` if the access key id and secret # access key are both set. def set? - !@native.nil? && - !access_key_id.nil? && - !access_key_id.empty? && - !secret_access_key.nil? && - !secret_access_key.empty? + native_set? end # Removing the secret access key from the default inspect string. @@ -82,21 +75,6 @@ def set? def inspect "#<#{self.class.name} access_key_id=#{access_key_id.inspect}>" end - - # Immediately release this instance's attachment to the underlying - # resources, without waiting for the garbage collector. - # Note that underlying resources will remain alive until nothing - # else is using them. - def release - return unless @native - - @native.free - @native = nil - end - - def self.on_release(native) - Aws::Crt::Native.credentials_release(native) - end end end end diff --git a/gems/aws-crt-auth/spec/credentials_spec.rb b/gems/aws-crt-auth/spec/credentials_spec.rb index 56c7f88..06e4ab1 100644 --- a/gems/aws-crt-auth/spec/credentials_spec.rb +++ b/gems/aws-crt-auth/spec/credentials_spec.rb @@ -6,8 +6,6 @@ module Aws module Crt module Auth #:nodoc: - UINT64_MAX = 18_446_744_073_709_551_615 - describe Credentials do describe '#initilize' do it 'raises an ArgumentError when missing access_key_id' do diff --git a/gems/aws-crt/lib/aws-crt.rb b/gems/aws-crt/lib/aws-crt.rb index cafc1e2..d1184c6 100644 --- a/gems/aws-crt/lib/aws-crt.rb +++ b/gems/aws-crt/lib/aws-crt.rb @@ -3,6 +3,7 @@ require_relative 'aws-crt/platforms' require_relative 'aws-crt/native' require_relative 'aws-crt/errors' +require_relative 'aws-crt/managed_native' require_relative 'aws-crt/io' # Top level Amazon Web Services (AWS) namespace diff --git a/gems/aws-crt/lib/aws-crt/io.rb b/gems/aws-crt/lib/aws-crt/io.rb index 8066fad..0ef1dbb 100644 --- a/gems/aws-crt/lib/aws-crt/io.rb +++ b/gems/aws-crt/lib/aws-crt/io.rb @@ -9,6 +9,9 @@ module IO # Classes that need to do async work will ask the EventLoopGroup # for an event-loop to use. class EventLoopGroup + include Aws::Crt::ManagedNative + native_destroy Aws::Crt::Native.method(:event_loop_group_release) + def initialize(max_threads = nil) unless max_threads.nil? || (max_threads.is_a?(Integer) && max_threads.positive?) @@ -18,26 +21,9 @@ def initialize(max_threads = nil) # Ruby uses nil to request default values, native code uses 0 max_threads = 0 if max_threads.nil? - native = Aws::Crt.call do + manage_native do Aws::Crt::Native.event_loop_group_new(max_threads) end - - @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) - end - - # Immediately release this instance's attachment to the underlying - # resources, without waiting for the garbage collector. - # Note that underlying resources will remain alive until nothing - # else is using them. - def release - return unless @native - - @native.free - @native = nil - end - - def self.on_release(native) - Aws::Crt::Native.event_loop_group_release(native) end end end diff --git a/gems/aws-crt/lib/aws-crt/managed_native.rb b/gems/aws-crt/lib/aws-crt/managed_native.rb new file mode 100644 index 0000000..bdb9b1d --- /dev/null +++ b/gems/aws-crt/lib/aws-crt/managed_native.rb @@ -0,0 +1,86 @@ +# frozen_string_literal: true + +module Aws + module Crt + # A mixin module for generic managed native functionality + # Example: + # + # class C + # include Aws::Crt::ManagedNative + # native_destroy Aws::Crt::Native.method(:test_struct_destroy) + # + # def initialize + # manage_native { Aws::Crt::Native::test_struct_new() } + # end + # + # def use_native + # Aws::Crt::Native::test_method(native) #use that getter for native + # end + # end + module ManagedNative + def self.included(sub_class) + sub_class.extend(ClassMethods) + end + + # expects a block that returns a :pointer to the native resource + # that this class manages + def manage_native(&block) + # check that a destructor has been registered + unless self.class.instance_variable_get('@destructor') + raise 'No native destructor registered. use native_destroy to ' \ + 'set the method used to cleanup the native object this ' \ + 'class manages.' + end + native = Aws::Crt.call { block.call } + @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) + end + + # @param [Boolean] safe (true) - raise an exception if the native object + # is not set (has been freed or never created) + # @return [FFI:Pointer] + def native(safe: true) + raise '@native is unset or has been freed.' if safe && !@native + + @native + end + + # @return [Boolean] + def native_set? + !!@native + end + + # Immediately release this instance's attachment to the underlying + # resources, without waiting for the garbage collector. + # Note that underlying resources will remain alive until nothing + # else is using them. + def release + return unless @native + + @native.free + @native = nil + end + + # ClassMethods for ManagedNative + module ClassMethods + # Register the method used to cleanup the native object this class + # manages. Must be a method, use object.method(:method_name). + # + # Example: + # native_destroy Aws::Crt::Native.method(:test_release) + def native_destroy(destructor) + unless destructor.is_a?(Method) + raise ArgumentError, 'destructor must be a Method. ' \ + 'Use object.method(:method_name)' + end + @destructor = destructor + end + + # Do not call directly + # method passed to FFI Autopointer to call the destructor + def on_release(native) + @destructor.call(native) + end + end + end + end +end diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index add4bbd..01b55f6 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -49,10 +49,10 @@ AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( const char *session_token, uint64_t expiration_timepoint_seconds); AWS_CRT_API void aws_crt_credentials_release(struct aws_credentials *credentials); -AWS_CRT_API const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials); -AWS_CRT_API const char *aws_crt_credentials_get_secret_access_key(struct aws_credentials *credentials); -AWS_CRT_API const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials); -AWS_CRT_API uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials); +AWS_CRT_API const char *aws_crt_credentials_get_access_key_id(const struct aws_credentials *credentials); +AWS_CRT_API const char *aws_crt_credentials_get_secret_access_key(const struct aws_credentials *credentials); +AWS_CRT_API const char *aws_crt_credentials_get_session_token(const struct aws_credentials *credentials); +AWS_CRT_API uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(const struct aws_credentials *credentials); AWS_EXTERN_C_END diff --git a/gems/aws-crt/native/src/credentials.c b/gems/aws-crt/native/src/credentials.c index c41ce57..d751edf 100644 --- a/gems/aws-crt/native/src/credentials.c +++ b/gems/aws-crt/native/src/credentials.c @@ -20,27 +20,21 @@ struct aws_credentials *aws_crt_credentials_new( aws_byte_cursor_from_c_str(secret_access_key), aws_byte_cursor_from_c_str(session_token), expiration_timepoint_seconds); - // return aws_credentials_new_from_string( - // allocator, - // aws_string_new_from_c_str(allocator, access_key_id), - // aws_string_new_from_c_str(allocator, secret_access_key), - // session_token == NULL ? NULL : aws_string_new_from_c_str(allocator, session_token), - // expiration_timepoint_seconds); } -const char *aws_crt_credentials_get_access_key_id(struct aws_credentials *credentials) { +const char *aws_crt_credentials_get_access_key_id(const struct aws_credentials *credentials) { return (char *)aws_credentials_get_access_key_id(credentials).ptr; } -const char *aws_crt_credentials_get_secret_access_key(struct aws_credentials *credentials) { +const char *aws_crt_credentials_get_secret_access_key(const struct aws_credentials *credentials) { return (char *)aws_credentials_get_secret_access_key(credentials).ptr; } -const char *aws_crt_credentials_get_session_token(struct aws_credentials *credentials) { +const char *aws_crt_credentials_get_session_token(const struct aws_credentials *credentials) { return (char *)aws_credentials_get_session_token(credentials).ptr; } -uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(struct aws_credentials *credentials) { +uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(const struct aws_credentials *credentials) { return aws_credentials_get_expiration_timepoint_seconds(credentials); } From a16f86c4b75870cb269c441951de67a2a124993a Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 30 Sep 2020 16:44:04 -0700 Subject: [PATCH 12/17] Use a ByteCursor struct for return types instead of :string --- .../lib/aws-crt-auth/credentials.rb | 13 ++++++------- gems/aws-crt/lib/aws-crt/native.rb | 19 +++++++++++++++---- gems/aws-crt/native/src/api.h | 6 +++--- gems/aws-crt/native/src/credentials.c | 12 ++++++------ 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index 4870885..af19ce1 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -4,7 +4,6 @@ module Aws module Crt module Auth # Utility class for Credentials. - # @api private class Credentials include Aws::Crt::ManagedNative native_destroy Aws::Crt::Native.method(:credentials_release) @@ -14,7 +13,7 @@ class Credentials # @param [String] access_key_id # @param [String] secret_access_key # @param [String] session_token (nil) - # @param [Time|int] expiration (nil) - Either a Time or if an int + # @param [Time|int] expiration (nil) - Either a Time or an int # seconds since unix epoch def initialize(access_key_id, secret_access_key, session_token = nil, expiration = nil) @@ -36,19 +35,19 @@ def initialize(access_key_id, secret_access_key, end end - # @return [String, nil] + # @return [String] def access_key_id - Aws::Crt::Native.credentials_get_access_key_id(native) + Aws::Crt::Native.credentials_get_access_key_id(native).to_s end - # @return [String, nil] + # @return [String] def secret_access_key - Aws::Crt::Native.credentials_get_secret_access_key(native) + Aws::Crt::Native.credentials_get_secret_access_key(native).to_s end # @return [String, nil] def session_token - Aws::Crt::Native.credentials_get_session_token(native) + Aws::Crt::Native.credentials_get_session_token(native).to_s end # @return [Time,nil] diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index 4718102..cfaa364 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -1,7 +1,6 @@ # frozen_string_literal: true require 'ffi' - module Aws module Crt # FFI Bindings to native CRT functions @@ -10,6 +9,18 @@ module Native ffi_lib [crt_bin_path(local_platform), 'libaws-crt'] + # aws_byte_cursor binding + class ByteCursor < FFI::Struct + layout :len, :size_t, + :ptr, :pointer + + def to_s + return unless (self[:len]).positive? && !(self[:ptr]).null? + + self[:ptr].read_string_length(self[:len]) + end + end + # Core API attach_function :init, :aws_crt_init, [], :void attach_function :last_error, :aws_crt_last_error, [], :int @@ -26,9 +37,9 @@ module Native # Auth API attach_function :credentials_new, :aws_crt_credentials_new, %i[string string string uint64], :pointer attach_function :credentials_release, :aws_crt_credentials_release, [:pointer], :void - attach_function :credentials_get_access_key_id, :aws_crt_credentials_get_access_key_id, [:pointer], :string - attach_function :credentials_get_secret_access_key, :aws_crt_credentials_get_secret_access_key, [:pointer], :string - attach_function :credentials_get_session_token, :aws_crt_credentials_get_session_token, [:pointer], :string + attach_function :credentials_get_access_key_id, :aws_crt_credentials_get_access_key_id, [:pointer], ByteCursor.by_value + attach_function :credentials_get_secret_access_key, :aws_crt_credentials_get_secret_access_key, [:pointer], ByteCursor.by_value + attach_function :credentials_get_session_token, :aws_crt_credentials_get_session_token, [:pointer], ByteCursor.by_value attach_function :credentials_get_expiration, :aws_crt_credentials_get_expiration_timepoint_seconds, [:pointer], :uint64 # Internal testing API diff --git a/gems/aws-crt/native/src/api.h b/gems/aws-crt/native/src/api.h index 01b55f6..102e065 100644 --- a/gems/aws-crt/native/src/api.h +++ b/gems/aws-crt/native/src/api.h @@ -49,9 +49,9 @@ AWS_CRT_API struct aws_credentials *aws_crt_credentials_new( const char *session_token, uint64_t expiration_timepoint_seconds); AWS_CRT_API void aws_crt_credentials_release(struct aws_credentials *credentials); -AWS_CRT_API const char *aws_crt_credentials_get_access_key_id(const struct aws_credentials *credentials); -AWS_CRT_API const char *aws_crt_credentials_get_secret_access_key(const struct aws_credentials *credentials); -AWS_CRT_API const char *aws_crt_credentials_get_session_token(const struct aws_credentials *credentials); +AWS_CRT_API struct aws_byte_cursor aws_crt_credentials_get_access_key_id(const struct aws_credentials *credentials); +AWS_CRT_API struct aws_byte_cursor aws_crt_credentials_get_secret_access_key(const struct aws_credentials *credentials); +AWS_CRT_API struct aws_byte_cursor aws_crt_credentials_get_session_token(const struct aws_credentials *credentials); AWS_CRT_API uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(const struct aws_credentials *credentials); AWS_EXTERN_C_END diff --git a/gems/aws-crt/native/src/credentials.c b/gems/aws-crt/native/src/credentials.c index d751edf..5888439 100644 --- a/gems/aws-crt/native/src/credentials.c +++ b/gems/aws-crt/native/src/credentials.c @@ -22,16 +22,16 @@ struct aws_credentials *aws_crt_credentials_new( expiration_timepoint_seconds); } -const char *aws_crt_credentials_get_access_key_id(const struct aws_credentials *credentials) { - return (char *)aws_credentials_get_access_key_id(credentials).ptr; +struct aws_byte_cursor aws_crt_credentials_get_access_key_id(const struct aws_credentials *credentials) { + return aws_credentials_get_access_key_id(credentials); } -const char *aws_crt_credentials_get_secret_access_key(const struct aws_credentials *credentials) { - return (char *)aws_credentials_get_secret_access_key(credentials).ptr; +struct aws_byte_cursor aws_crt_credentials_get_secret_access_key(const struct aws_credentials *credentials) { + return aws_credentials_get_secret_access_key(credentials); } -const char *aws_crt_credentials_get_session_token(const struct aws_credentials *credentials) { - return (char *)aws_credentials_get_session_token(credentials).ptr; +struct aws_byte_cursor aws_crt_credentials_get_session_token(const struct aws_credentials *credentials) { + return aws_credentials_get_session_token(credentials); } uint64_t aws_crt_credentials_get_expiration_timepoint_seconds(const struct aws_credentials *credentials) { From 4a23229691b2cc9a181029454671b5f77dc299fd Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Wed, 30 Sep 2020 16:52:10 -0700 Subject: [PATCH 13/17] Fix jruby failures from missing FFI methods... --- gems/aws-crt/lib/aws-crt/native.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index cfaa364..c585f59 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -17,7 +17,7 @@ class ByteCursor < FFI::Struct def to_s return unless (self[:len]).positive? && !(self[:ptr]).null? - self[:ptr].read_string_length(self[:len]) + self[:ptr].get_string(0, self[:len]) end end From 6ff266f91f0307636719294ff6cbd87b1ef31eaa Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Thu, 1 Oct 2020 09:39:25 -0700 Subject: [PATCH 14/17] Extend attach_function to add error wrapping + custom naming --- .github/workflows/ci.yml | 4 +- .../lib/aws-crt-auth/credentials.rb | 4 +- gems/aws-crt/lib/aws-crt.rb | 14 ----- gems/aws-crt/lib/aws-crt/managed_native.rb | 2 +- gems/aws-crt/lib/aws-crt/native.rb | 57 +++++++++++++++---- gems/aws-crt/spec/crt_spec.rb | 19 ------- gems/aws-crt/spec/errors_spec.rb | 6 +- gems/aws-crt/spec/native_spec.rb | 34 +++++++++++ gems/aws-crt/spec/spec_helper.rb | 2 +- 9 files changed, 90 insertions(+), 52 deletions(-) delete mode 100644 gems/aws-crt/spec/crt_spec.rb create mode 100644 gems/aws-crt/spec/native_spec.rb diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d5420cd..91ed7c8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,7 +21,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: [2.5, 2.6, 2.7, jruby, truffleruby] + ruby: [2.5, 2.6, 2.7, jruby] runs-on: ubuntu-latest steps: - uses: ruby/setup-ruby@v1 @@ -72,7 +72,7 @@ jobs: fail-fast: false matrix: os: [macos] - ruby: [2.5, 2.6, 2.7, jruby, truffleruby] + ruby: [2.5, 2.6, 2.7, jruby] runs-on: macos-latest steps: - uses: ruby/setup-ruby@v1 diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index af19ce1..cc111ec 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -52,7 +52,9 @@ def session_token # @return [Time,nil] def expiration - exp = Aws::Crt::Native.credentials_get_expiration(native) + exp = Aws::Crt::Native.credentials_get_expiration_timepoint_seconds!( + native + ) return if exp == UINT64_MAX Time.at(exp) diff --git a/gems/aws-crt/lib/aws-crt.rb b/gems/aws-crt/lib/aws-crt.rb index d1184c6..3bf052b 100644 --- a/gems/aws-crt/lib/aws-crt.rb +++ b/gems/aws-crt/lib/aws-crt.rb @@ -12,19 +12,5 @@ module Aws module Crt # Ensure native init() is called when gem loads Aws::Crt::Native.init - - # Invoke native call, and raise exception if it failed - def self.call - res = yield - # functions that return void cannot fail - return unless res - - # for functions that return int, non-zero indicates failure - Errors.raise_last_error if res.is_a?(Integer) && res != 0 - - # for functions that return pointer, NULL indicates failure - Errors.raise_last_error if res.is_a?(FFI::Pointer) && res.null? - res - end end end diff --git a/gems/aws-crt/lib/aws-crt/managed_native.rb b/gems/aws-crt/lib/aws-crt/managed_native.rb index bdb9b1d..e1f8081 100644 --- a/gems/aws-crt/lib/aws-crt/managed_native.rb +++ b/gems/aws-crt/lib/aws-crt/managed_native.rb @@ -31,7 +31,7 @@ def manage_native(&block) 'set the method used to cleanup the native object this ' \ 'class manages.' end - native = Aws::Crt.call { block.call } + native = block.call @native = FFI::AutoPointer.new(native, self.class.method(:on_release)) end diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index c585f59..0a7167f 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -28,23 +28,58 @@ def to_s attach_function :error_name, :aws_crt_error_name, [:int], :string attach_function :error_debug_str, :aws_crt_error_debug_str, [:int], :string attach_function :reset_error, :aws_crt_reset_error, [], :void - attach_function :global_thread_creator_shutdown_wait_for, :aws_crt_global_thread_creator_shutdown_wait_for, [:uint32], :int + + # This MUST follow definitions of core error functions since it relies on them + # And error functions should NOT be wrapped. + # + # Overridden for three purposes. + # + # 1. Allows us to only supply the aws_crt C name, and converts it removes + # the aws_crt. + # 2. Wraps the call in an error-raise checker. + # 3. Creates a bang method that does not do automatic error checking. + def self.attach_function(c_name, params, returns, options = {}) + ruby_name = c_name.to_s.sub(/aws_crt_/, '').to_sym + bang_name = "#{ruby_name}!" + + super(ruby_name, c_name, params, returns, options) + alias_method(bang_name, ruby_name) + + define_method(ruby_name) do |*args, &block| + res = public_send(bang_name, *args, &block) + # functions that return void cannot fail + return unless res + + # for functions that return int, non-zero indicates failure + Errors.raise_last_error if res.is_a?(Integer) && res != 0 + + # for functions that return pointer, NULL indicates failure + Errors.raise_last_error if res.is_a?(FFI::Pointer) && res.null? + + res + end + + module_function ruby_name + module_function bang_name + end + + attach_function :aws_crt_global_thread_creator_shutdown_wait_for, [:uint32], :int # IO API - attach_function :event_loop_group_new, :aws_crt_event_loop_group_new, [:uint16], :pointer - attach_function :event_loop_group_release, :aws_crt_event_loop_group_release, [:pointer], :void + attach_function :aws_crt_event_loop_group_new, [:uint16], :pointer + attach_function :aws_crt_event_loop_group_release, [:pointer], :void # Auth API - attach_function :credentials_new, :aws_crt_credentials_new, %i[string string string uint64], :pointer - attach_function :credentials_release, :aws_crt_credentials_release, [:pointer], :void - attach_function :credentials_get_access_key_id, :aws_crt_credentials_get_access_key_id, [:pointer], ByteCursor.by_value - attach_function :credentials_get_secret_access_key, :aws_crt_credentials_get_secret_access_key, [:pointer], ByteCursor.by_value - attach_function :credentials_get_session_token, :aws_crt_credentials_get_session_token, [:pointer], ByteCursor.by_value - attach_function :credentials_get_expiration, :aws_crt_credentials_get_expiration_timepoint_seconds, [:pointer], :uint64 + attach_function :aws_crt_credentials_new, %i[string string string uint64], :pointer + attach_function :aws_crt_credentials_release, [:pointer], :void + attach_function :aws_crt_credentials_get_access_key_id, [:pointer], ByteCursor.by_value + attach_function :aws_crt_credentials_get_secret_access_key, [:pointer], ByteCursor.by_value + attach_function :aws_crt_credentials_get_session_token, [:pointer], ByteCursor.by_value + attach_function :aws_crt_credentials_get_expiration_timepoint_seconds, [:pointer], :uint64 # Internal testing API - attach_function :test_error, :aws_crt_test_error, [:int], :int - attach_function :test_pointer_error, :aws_crt_test_pointer_error, [], :pointer + attach_function :aws_crt_test_error, [:int], :int + attach_function :aws_crt_test_pointer_error, [], :pointer end end end diff --git a/gems/aws-crt/spec/crt_spec.rb b/gems/aws-crt/spec/crt_spec.rb deleted file mode 100644 index 960c1ed..0000000 --- a/gems/aws-crt/spec/crt_spec.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -require_relative 'spec_helper' - -describe Aws::Crt do - describe '#call' do - it 'raises an error when called on a function that returns an int' do - expect do - Aws::Crt.call { Aws::Crt::Native.test_error(3) } - end.to raise_error(Aws::Crt::Error) - end - - it 'raises an error when called on a function that returns a pointer' do - expect do - Aws::Crt.call { Aws::Crt::Native.test_pointer_error } - end.to raise_error(NoMemoryError) - end - end -end diff --git a/gems/aws-crt/spec/errors_spec.rb b/gems/aws-crt/spec/errors_spec.rb index 500c433..8385a45 100644 --- a/gems/aws-crt/spec/errors_spec.rb +++ b/gems/aws-crt/spec/errors_spec.rb @@ -5,21 +5,21 @@ describe Aws::Crt::Errors do describe '.raise_last_error' do it 'translates and raises the last error' do - Aws::Crt::Native.test_error(3) # generate an error + Aws::Crt::Native.test_error!(3) # generate an error expect do Aws::Crt::Errors.raise_last_error end.to raise_error(Aws::Crt::Error) end it 'does not raise when no error' do - Aws::Crt::Native.test_error(0) # success + Aws::Crt::Native.test_error!(0) # success expect do Aws::Crt::Errors.raise_last_error end.not_to raise_error end it 'resets the error after raising it' do - Aws::Crt::Native.test_error(3) # raise error + Aws::Crt::Native.test_error!(3) # raise error expect do Aws::Crt::Errors.raise_last_error end.to raise_error(Aws::Crt::Error) diff --git a/gems/aws-crt/spec/native_spec.rb b/gems/aws-crt/spec/native_spec.rb new file mode 100644 index 0000000..49ad05f --- /dev/null +++ b/gems/aws-crt/spec/native_spec.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +require_relative 'spec_helper' + +describe Aws::Crt::Native do + describe '.attach_function' do + it 'removes the aws_crt_ prefix from C functions' do + expect(Aws::Crt::Native).to respond_to(:test_error) + expect(Aws::Crt::Native).not_to respond_to(:aws_crt_test_error) + end + + it 'creates a ! version of the function' do + expect(Aws::Crt::Native).to respond_to(:test_error!) + end + + it 'raises an error when called on a function that returns an int' do + expect do + Aws::Crt::Native.test_error(3) + end.to raise_error(Aws::Crt::Error) + end + + it 'raises an error when called on a function that returns a pointer' do + expect do + Aws::Crt::Native.test_pointer_error + end.to raise_error(NoMemoryError) + end + + it 'the ! function does not raise an error' do + expect do + Aws::Crt::Native.test_error!(3) + end.not_to raise_error + end + end +end diff --git a/gems/aws-crt/spec/spec_helper.rb b/gems/aws-crt/spec/spec_helper.rb index bdbe1c4..0a5c549 100644 --- a/gems/aws-crt/spec/spec_helper.rb +++ b/gems/aws-crt/spec/spec_helper.rb @@ -15,5 +15,5 @@ def garbage_collect_is_immediate? def check_for_clean_shutdown ObjectSpace.garbage_collect - Aws::Crt.call { Aws::Crt::Native.global_thread_creator_shutdown_wait_for(10) } + Aws::Crt::Native.global_thread_creator_shutdown_wait_for(10) end From 13fb4f76ba732d41ae29f3af7d37bc2b811ef79c Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Thu, 1 Oct 2020 09:48:57 -0700 Subject: [PATCH 15/17] Remove credential SDK interface methods --- gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb | 11 ----------- gems/aws-crt-auth/spec/credentials_spec.rb | 12 ------------ 2 files changed, 23 deletions(-) diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb index cc111ec..1c62941 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth/credentials.rb @@ -60,17 +60,6 @@ def expiration Time.at(exp) end - # @return [Credentials] - def credentials - self - end - - # @return [Boolean] Returns `true` if the access key id and secret - # access key are both set. - def set? - native_set? - end - # Removing the secret access key from the default inspect string. # @api private def inspect diff --git a/gems/aws-crt-auth/spec/credentials_spec.rb b/gems/aws-crt-auth/spec/credentials_spec.rb index 06e4ab1..a86b232 100644 --- a/gems/aws-crt-auth/spec/credentials_spec.rb +++ b/gems/aws-crt-auth/spec/credentials_spec.rb @@ -61,18 +61,6 @@ module Auth #:nodoc: end end - describe '#set?' do - it 'returns true when the key and secret are both non nil values' do - expect(Credentials.new('akid', 'secret').set?).to be(true) - end - - it 'returns false after the credentials have been released' do - creds = Credentials.new('akid', 'secret') - creds.release - expect(creds.set?).to be(false) - end - end - describe '#inspect' do let(:creds) { Credentials.new('akid', 'secret', 'token') } From 5db602e877da5faf9273e9d005d923aaa9af154b Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Thu, 8 Oct 2020 08:42:43 -0700 Subject: [PATCH 16/17] Cleanups from PR - remove signer interface --- gems/aws-crt-auth/lib/aws-crt-auth/signer.rb | 37 -------------------- gems/aws-crt-auth/spec/signer_spec.rb | 9 ----- gems/aws-crt/lib/aws-crt/native.rb | 34 ++++++++++-------- 3 files changed, 19 insertions(+), 61 deletions(-) delete mode 100644 gems/aws-crt-auth/lib/aws-crt-auth/signer.rb delete mode 100644 gems/aws-crt-auth/spec/signer_spec.rb diff --git a/gems/aws-crt-auth/lib/aws-crt-auth/signer.rb b/gems/aws-crt-auth/lib/aws-crt-auth/signer.rb deleted file mode 100644 index da111d3..0000000 --- a/gems/aws-crt-auth/lib/aws-crt-auth/signer.rb +++ /dev/null @@ -1,37 +0,0 @@ -# frozen_string_literal: true - -module Aws - module Crt - module Auth - # Utility class for creating AWS signature version 4 signature. - class Signer - def initialize(options = {}) - @service = extract_service(options) - @region = extract_region(options) - @credentials_provider = extract_credentials_provider(options) - @unsigned_headers = Set.new((options.fetch(:unsigned_headers, [])) - .map(&:downcase)) - @unsigned_headers << 'authorization' - @unsigned_headers << 'x-amzn-trace-id' - @unsigned_headers << 'expect' - %i[uri_escape_path apply_checksum_header].each do |opt| - instance_variable_set("@#{opt}", - options.key?(opt) ? options[:opt] : true) - end - end - - def sign_request(request) - # TODO - end - - def sign_event(prior_signature, payload, encoder) - # TODO - end - - def presign_url(options) - # TODO - end - end - end - end -end diff --git a/gems/aws-crt-auth/spec/signer_spec.rb b/gems/aws-crt-auth/spec/signer_spec.rb deleted file mode 100644 index 253e7d7..0000000 --- a/gems/aws-crt-auth/spec/signer_spec.rb +++ /dev/null @@ -1,9 +0,0 @@ -# frozen_string_literal: true - -require_relative 'spec_helper' - -require 'tempfile' -require 'base64' - -describe Aws::Crt::Auth::Signer do -end diff --git a/gems/aws-crt/lib/aws-crt/native.rb b/gems/aws-crt/lib/aws-crt/native.rb index 0a7167f..9b202e6 100644 --- a/gems/aws-crt/lib/aws-crt/native.rb +++ b/gems/aws-crt/lib/aws-crt/native.rb @@ -17,29 +17,25 @@ class ByteCursor < FFI::Struct def to_s return unless (self[:len]).positive? && !(self[:ptr]).null? - self[:ptr].get_string(0, self[:len]) + self[:ptr].read_string(self[:len]) end end - # Core API - attach_function :init, :aws_crt_init, [], :void - attach_function :last_error, :aws_crt_last_error, [], :int - attach_function :error_str, :aws_crt_error_str, [:int], :string - attach_function :error_name, :aws_crt_error_name, [:int], :string - attach_function :error_debug_str, :aws_crt_error_debug_str, [:int], :string - attach_function :reset_error, :aws_crt_reset_error, [], :void - - # This MUST follow definitions of core error functions since it relies on them - # And error functions should NOT be wrapped. - # - # Overridden for three purposes. + # Extends FFI::attach_function # - # 1. Allows us to only supply the aws_crt C name, and converts it removes + # 1. Allows us to only supply the aws_crt C name and removes # the aws_crt. - # 2. Wraps the call in an error-raise checker. + # 2. Wraps the call in an error-raise checker (unless options[:raise] + # = false) # 3. Creates a bang method that does not do automatic error checking. def self.attach_function(c_name, params, returns, options = {}) ruby_name = c_name.to_s.sub(/aws_crt_/, '').to_sym + raise_errors = options.fetch(:raise, true) + options.delete(:raise) + unless raise_errors + return super(ruby_name, c_name, params, returns, options) + end + bang_name = "#{ruby_name}!" super(ruby_name, c_name, params, returns, options) @@ -63,6 +59,14 @@ def self.attach_function(c_name, params, returns, options = {}) module_function bang_name end + # Core API + attach_function :aws_crt_init, [], :void, raise: false + attach_function :aws_crt_last_error, [], :int, raise: false + attach_function :aws_crt_error_str, [:int], :string, raise: false + attach_function :aws_crt_error_name, [:int], :string, raise: false + attach_function :aws_crt_error_debug_str, [:int], :string, raise: false + attach_function :aws_crt_reset_error, [], :void, raise: false + attach_function :aws_crt_global_thread_creator_shutdown_wait_for, [:uint32], :int # IO API From 73ec9f37abc6fef084e99e397f2530f0c3600427 Mon Sep 17 00:00:00 2001 From: Alex Woods Date: Thu, 8 Oct 2020 08:50:18 -0700 Subject: [PATCH 17/17] Rubocop cleanup --- .rubocop.yml | 3 +++ gems/aws-crt-auth/lib/aws-crt-auth.rb | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.rubocop.yml b/.rubocop.yml index 770e4ca..9e3c81b 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -26,6 +26,9 @@ Metrics/MethodLength: Exclude: - 'gems/aws-crt/ext/compile.rb' +Metrics/AbcSize: + Max: 20 + Naming/FileName: Exclude: - 'gems/aws-crt/lib/aws-crt.rb' diff --git a/gems/aws-crt-auth/lib/aws-crt-auth.rb b/gems/aws-crt-auth/lib/aws-crt-auth.rb index 736774e..ee45cba 100644 --- a/gems/aws-crt-auth/lib/aws-crt-auth.rb +++ b/gems/aws-crt-auth/lib/aws-crt-auth.rb @@ -2,7 +2,6 @@ require 'aws-crt' require_relative 'aws-crt-auth/credentials' -require_relative 'aws-crt-auth/signer' module Aws module Crt