diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec60e5c4..d311f2eb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,15 +3,13 @@ name: Build on: pull_request: push: - branches: - - master jobs: build: strategy: matrix: platform: [ubuntu-latest] - otp-version: [24, 25, 26, 27] + otp-version: [25, 26, 27] runs-on: ${{ matrix.platform }} container: image: erlang:${{ matrix.otp-version }} @@ -27,20 +25,6 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 - - name: Cache Hex packages - uses: actions/cache@v4 - with: - path: ~/.cache/rebar3/hex/hexpm/packages - key: ${{ runner.os }}-hex-${{ hashFiles(format('{0}{1}', github.workspace, '/rebar.lock')) }} - restore-keys: | - ${{ runner.os }}-hex- - - name: Cache Dialyzer PLTs - uses: actions/cache@v4 - with: - path: ~/.cache/rebar3/rebar3_*.plt - key: ${{ runner.os }}-dialyzer-${{ hashFiles(format('{0}{1}', github.workspace, '/rebar.config')) }} - restore-keys: | - ${{ runner.os }}-dialyzer- - name: Compile run: rebar3 compile - name: Run EUnit Tests @@ -52,14 +36,8 @@ jobs: S3MOCK_HOST: s3mock - name: Check app calls run: rebar3 check_app_calls - - name: Create Cover Reports - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: rebar3 cover - - name: Produce Documentation - run: rebar3 edoc || true - name: Produce Documentation - run: rebar3 ex_doc || true + run: rebar3 ex_doc - name: Publish Documentation uses: actions/upload-artifact@v4 with: diff --git a/README.md b/README.md index 8eb48ce8..9f5f6ea5 100644 --- a/README.md +++ b/README.md @@ -10,13 +10,14 @@ :rocket: Create, configure, and manage AWS services from Erlang code. :rocket: ## Features +This repo only holds generated code :exclamation: All non-generated code is included as part of [aws_beam_core](https://github.com/aws-beam/aws_beam_core). +Any changes that need to be made should hence be made through [aws_beam_core](https://github.com/aws-beam/aws_beam_core) or [aws-codegen](https://github.com/aws-beam/aws-codegen) :exclamation: +* Completely generated by [aws-codegen](https://github.com/aws-beam/aws-codegen) from the same JSON descriptions of AWS services used to build the AWS SDKs (See: [aws-sdk-go-v2](https://github.com/aws/aws-sdk-go-v2)) * A clean API separated per service. One module per service. -* Support for most of the AWS services. -* Generated by [aws-codegen](https://github.com/aws-beam/aws-codegen) using the - same JSON descriptions of AWS services used to build the - [AWS SDK for Go](https://github.com/aws/aws-sdk-go/tree/master/models/apis). +* Support for most (almost all!) of the AWS services. * Documentation updated from the official AWS docs. +* Type specs generated from the same JSON descriptions as the SDK ## Usage @@ -54,8 +55,9 @@ through the `aws_s3_presigned_url` module. ``` ### AWS RDS IAM Token Creation -Support for creating IAM Tokens (more info [here](https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.Connecting.html)) has been added as part of the `aws_rds_iam_token` module. +Support for creating IAM Tokens (more info here) has been added as part of the aws_rds_iam_token module as part of [aws_beam_core](https://github.com/aws-beam/aws_beam_core). This allows for easy creation of RDS/Aurora tokens to be used for IAM based authentication instead of username/password combination. + ```erlang > Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, <<"Token">>, <<"eu-west-1">>). [...] @@ -65,6 +67,17 @@ This allows for easy creation of RDS/Aurora tokens to be used for IAM based auth This token can subsequently be used to connect to the database over IAM. +### AWS S3 Presigned Url +Support for Presigning S3 urls has been added as part of the aws_s3_presigned_url module as part of [aws_beam_core](https://github.com/aws-beam/aws_beam_core). +This allows generating either a get or put presigned s3 url, +which can be used by external clients such as cURL to access (get/put) the object in question. +```erlang +> Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, <<"Token">>, <<"eu-west-1">>). +[...] +> {ok, Url} = aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, <<"bucket">>, <<"key">>) +[...] +``` + ### retry options Each API which takes `Options` allows a `retry_options` key and can allow for automatic retries. @@ -79,7 +92,7 @@ This implementation is based on [AWS: Exponential Backoff And Jitter](https://aw Simply add the library to your `rebar.config`: ```erlang -{deps, [{aws, "1.0.0", {pkg, aws_erlang}}]}. +{deps, [{aws, "1.1.0", {pkg, aws_erlang}}]}. ``` ## Obtaining Credentials @@ -103,11 +116,7 @@ Here is an example on how to obtain credentials: , secret_access_key := SecretAccessKey } = Credentials. ``` -The `aws_credentials` application can be installed by adding the following to your `rebar.config`: - -```erlang -{deps, [{aws_credentials, "0.1.10"}]}. -``` +The `aws_credentials` application is part of [aws_beam_core](https://github.com/aws-beam/aws_beam_core) and included in this package. ## Development @@ -116,13 +125,7 @@ The service-specific modules are generated using the [aws-codegen](https://githu The rest of the code is manually written and used as support for the generated code. ## Documentation - -### Check it Online - -* [Hex Docs](https://hexdocs.pm/aws_erlang/) - -### Build it locally - +Unfortunately the docs generated are too big for hexdocs.pm. Hence, the docs can be generated locally using: ```bash $ rebar3 ex_doc ``` diff --git a/rebar.config b/rebar.config index 13b7ca35..365399d7 100644 --- a/rebar.config +++ b/rebar.config @@ -1,7 +1,7 @@ {erl_opts, [nowarn_unused_type, debug_info, {d, maps_support}]}. {deps, [{hackney, "1.18.0"}, {jsx, "3.0.0"}, - {aws_signature, "0.3.3"} + {aws_beam_core, "1.0.0"} ]}. {project_plugins, [rebar3_hex, rebar3_ex_doc, rebar3_check_app_calls]}. {hex, [{doc, #{provider => ex_doc}}]}. diff --git a/rebar.lock b/rebar.lock index 6046cf9b..4b3621a5 100644 --- a/rebar.lock +++ b/rebar.lock @@ -1,8 +1,12 @@ {"1.2.0", -[{<<"aws_signature">>,{pkg,<<"aws_signature">>,<<"0.3.3">>},0}, +[{<<"aws_beam_core">>,{pkg,<<"aws_beam_core">>,<<"1.0.0">>},0}, + {<<"aws_credentials">>,{pkg,<<"aws_credentials">>,<<"0.3.2">>},1}, + {<<"aws_signature">>,{pkg,<<"aws_signature">>,<<"0.3.3">>},0}, {<<"certifi">>,{pkg,<<"certifi">>,<<"2.5.2">>},1}, + {<<"eini">>,{pkg,<<"eini_beam">>,<<"2.2.4">>},2}, {<<"hackney">>,{pkg,<<"hackney">>,<<"1.16.0">>},0}, {<<"idna">>,{pkg,<<"idna">>,<<"6.0.1">>},1}, + {<<"iso8601">>,{pkg,<<"iso8601">>,<<"1.3.4">>},2}, {<<"jsx">>,{pkg,<<"jsx">>,<<"3.0.0">>},0}, {<<"metrics">>,{pkg,<<"metrics">>,<<"1.0.1">>},1}, {<<"mimerl">>,{pkg,<<"mimerl">>,<<"1.2.0">>},1}, @@ -11,10 +15,14 @@ {<<"unicode_util_compat">>,{pkg,<<"unicode_util_compat">>,<<"0.5.0">>},2}]}. [ {pkg_hash,[ + {<<"aws_beam_core">>, <<"F5BAD0147038A0B921844B530A8D937D8BF569B5E1FD0658FA0A6586EB98375C">>}, + {<<"aws_credentials">>, <<"BA2CCEE4EC6DCB5426CF71830B7AFD73795B1F19655F401D4401015B468FEC6F">>}, {<<"aws_signature">>, <<"5844BEE0D3CC42EEFD21D236BBFAA8AA9B16E2F2B7EE79EDAECB321DB3FB6ADF">>}, {<<"certifi">>, <<"B7CFEAE9D2ED395695DD8201C57A2D019C0C43ECAF8B8BCB9320B40D6662F340">>}, + {<<"eini">>, <<"02143B1DCE4DDA4243248E7D9B3D8274B8D9F5A666445E3D868E2CCE79E4FF22">>}, {<<"hackney">>, <<"5096AC8E823E3A441477B2D187E30DD3FFF1A82991A806B2003845CE72CE2D84">>}, {<<"idna">>, <<"1D038FB2E7668CE41FBF681D2C45902E52B3CB9E9C77B55334353B222C2EE50C">>}, + {<<"iso8601">>, <<"7B1F095F86F6CF65E1E5A77872E8E8BF69BD58D4C3A415B3F77D9CC9423ECBB9">>}, {<<"jsx">>, <<"20A170ABD4335FC6DB24D5FAD1E5D677C55DADF83D1B20A8A33B5FE159892A39">>}, {<<"metrics">>, <<"25F094DEA2CDA98213CECC3AEFF09E940299D950904393B2A29D191C346A8486">>}, {<<"mimerl">>, <<"67E2D3F571088D5CFD3E550C383094B47159F3EEE8FFA08E64106CDF5E981BE3">>}, @@ -22,10 +30,14 @@ {<<"ssl_verify_fun">>, <<"CF344F5692C82D2CD7554F5EC8FD961548D4FD09E7D22F5B62482E5AEAEBD4B0">>}, {<<"unicode_util_compat">>, <<"8516502659002CEC19E244EBD90D312183064BE95025A319A6C7E89F4BCCD65B">>}]}, {pkg_hash_ext,[ + {<<"aws_beam_core">>, <<"421C24834B210A10698ADB29282FF96777C711D85AC57FE6DF9C39DFB2E1FD58">>}, + {<<"aws_credentials">>, <<"2E748626A935A7A544647FB79D7054F38DB8BF378978542C962CCBEAB387387B">>}, {<<"aws_signature">>, <<"87E8F42B8E49002AA8D0350A71D13D69EA91B9AFB4CA9B526AE36DB1D585C924">>}, {<<"certifi">>, <<"3B3B5F36493004AC3455966991EAF6E768CE9884693D9968055AEEEB1E575040">>}, + {<<"eini">>, <<"12DE479D144B19E09BB92BA202A7EA716739929AFDF9DFF01AD802E2B1508471">>}, {<<"hackney">>, <<"3BF0BEBBD5D3092A3543B783BF065165FA5D3AD4B899B836810E513064134E18">>}, {<<"idna">>, <<"A02C8A1C4FD601215BB0B0324C8A6986749F807CE35F25449EC9E69758708122">>}, + {<<"iso8601">>, <<"A334469C07F1C219326BC891A95F5EEC8EB12DD8071A3FFF56A7843CB20FAE34">>}, {<<"jsx">>, <<"37BECA0435F5CA8A2F45F76A46211E76418FBEF80C36F0361C249FC75059DC6D">>}, {<<"metrics">>, <<"69B09ADDDC4F74A40716AE54D140F93BEB0FB8978D8636EADED0C31B6F099F16">>}, {<<"mimerl">>, <<"F278585650AA581986264638EBF698F8BB19DF297F66AD91B18910DFC6E19323">>}, diff --git a/src/aws.app.src b/src/aws.app.src index 38707a54..0ac2591e 100644 --- a/src/aws.app.src +++ b/src/aws.app.src @@ -5,11 +5,10 @@ , { applications , [ kernel , stdlib - , xmerl , crypto , hackney , jsx - , aws_signature + , aws_beam_core ] } , {env,[]} diff --git a/src/aws_client.erl b/src/aws_client.erl deleted file mode 100644 index 1cbc5ee8..00000000 --- a/src/aws_client.erl +++ /dev/null @@ -1,215 +0,0 @@ --module(aws_client). - --export([ make_client/0 - , make_client/1 - , make_client/3 - , make_temporary_client/4 - , make_local_client/3 - , make_local_client/4 - ]). - --export([ access_key_id/1 - , secret_access_key/1 - , token/1 - , region/1 - , endpoint/1 - , proto/1 - , port/1 - , service/1 - ]). - --type access_key_id() :: binary(). --type secret_access_key() :: binary(). --type region() :: binary(). --type token() :: binary(). --type http_port() :: binary(). --type proto() :: binary(). --type service() :: binary(). --type endpoint() :: binary(). --opaque aws_client() :: map(). - --export_type([access_key_id/0, secret_access_key/0, region/0, - token/0, aws_client/0]). - --define(AWS_ACCESS_KEY_ID, "AWS_ACCESS_KEY_ID"). --define(AWS_SECRET_ACCESS_KEY, "AWS_SECRET_ACCESS_KEY"). --define(AWS_SESSION_TOKEN, "AWS_SESSION_TOKEN"). --define(AWS_DEFAULT_REGION, "AWS_DEFAULT_REGION"). - -%%==================================================================== -%% API -%%==================================================================== - --spec make_client() -> aws_client(). -make_client() -> - case env(?AWS_DEFAULT_REGION) of - undefined -> error({missing_region, ?AWS_DEFAULT_REGION}); - Region -> make_client(Region) - end. - --spec make_client(region()) -> aws_client(). -make_client(Region) -> - case - { env(?AWS_ACCESS_KEY_ID) - , env(?AWS_SECRET_ACCESS_KEY) - , env(?AWS_SESSION_TOKEN) - } - of - {undefined, _, _} -> - error({missing_credentials, ?AWS_ACCESS_KEY_ID}); - {_, undefined, _} -> - error({missing_credentials, ?AWS_SECRET_ACCESS_KEY}); - {AccessKeyID, SecretAccessKey, undefined} -> - make_client(AccessKeyID, SecretAccessKey, Region); - {AccessKeyID, SecretAccessKey, Token} -> - make_temporary_client(AccessKeyID, SecretAccessKey, Token, Region) - end. - --spec make_client(access_key_id(), secret_access_key(), region()) -> - aws_client(). -make_client(AccessKeyID, SecretAccessKey, Region) - when is_binary(AccessKeyID), is_binary(SecretAccessKey), is_binary(Region) -> - #{access_key_id => AccessKeyID, - secret_access_key => SecretAccessKey, - region => Region, - endpoint => <<"amazonaws.com">>, - proto => <<"https">>, - port => <<"443">>, - service => undefined}. - --spec make_temporary_client(access_key_id(), secret_access_key(), token(), region()) -> - aws_client(). -make_temporary_client(AccessKeyID, SecretAccessKey, Token, Region) - when is_binary(AccessKeyID), is_binary(SecretAccessKey), - is_binary(Token), is_binary(Region) -> - #{access_key_id => AccessKeyID, - secret_access_key => SecretAccessKey, - token => Token, - region => Region, - endpoint => <<"amazonaws.com">>, - proto => <<"https">>, - port => <<"443">>, - service => undefined}. - --spec make_local_client(access_key_id(), secret_access_key(), http_port()) -> - aws_client(). - make_local_client(AccessKeyID, SecretAccessKey, Port) - when is_binary(AccessKeyID), is_binary(SecretAccessKey), is_binary(Port) -> - make_local_client(AccessKeyID, SecretAccessKey, Port, <<"localhost">>). - --spec make_local_client(access_key_id(), secret_access_key(), http_port(), endpoint()) -> - aws_client(). -make_local_client(AccessKeyID, SecretAccessKey, Port, Endpoint) - when is_binary(Endpoint) -> - #{access_key_id => AccessKeyID, - secret_access_key => SecretAccessKey, - region => <<"local">>, - endpoint => Endpoint, - proto => <<"http">>, - port => Port, - service => undefined}. - --spec access_key_id(aws_client()) -> access_key_id(). -access_key_id(#{access_key_id := AccessKeyId} = _Client) -> - AccessKeyId. - --spec secret_access_key(aws_client()) -> secret_access_key(). -secret_access_key(#{secret_access_key := SecretAccessKey} = _Client) -> - SecretAccessKey. - --spec token(aws_client()) -> token(). -token(#{token := Token} = _Client) -> - Token; -token(Client) when is_map(Client) -> - undefined. - --spec region(aws_client()) -> region(). -region(#{region := Region} = _Client) -> - Region. - --spec endpoint(aws_client()) -> endpoint(). -endpoint(#{endpoint := Endpoint} = _Client) -> - Endpoint. - --spec proto(aws_client()) -> proto(). -proto(#{proto := Proto} = _Client) -> - Proto. - --spec port(aws_client()) -> port(). -port(#{port := Port} = _Client) -> - Port. - --spec service(aws_client()) -> service(). -service(#{service := Service} = _Client) -> - Service. - -%%==================================================================== -%% Helper functions -%%==================================================================== - --spec env(string()) -> binary() | undefined. -env(Name) -> - case os:getenv(Name) of - false -> undefined; - Value -> list_to_binary(Value) - end. - -%%==================================================================== -%% Unit tests -%%==================================================================== - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -%% make_client/0 returns a clienturation map with default values. -make_client_test() -> - ?assertEqual(#{access_key_id => <<"access-key-id">>, - endpoint => <<"amazonaws.com">>, - port => <<"443">>, - proto => <<"https">>, - region => <<"region">>, - secret_access_key => <<"secret-access-key">>, - service => undefined}, - make_client(<<"access-key-id">>, <<"secret-access-key">>, - <<"region">>)). - -make_temporary_client_test() -> - ?assertEqual(#{access_key_id => <<"access-key-id">>, - endpoint => <<"amazonaws.com">>, - port => <<"443">>, - proto => <<"https">>, - region => <<"region">>, - secret_access_key => <<"secret-access-key">>, - service => undefined, - token => <<"some-token">>}, - make_temporary_client(<<"access-key-id">>, - <<"secret-access-key">>, - <<"some-token">>, - <<"region">>)). - -make_local_client_3_test() -> - ?assertEqual(#{access_key_id => <<"access-key-id">>, - port => <<"8000">>, - proto => <<"http">>, - region => <<"local">>, - endpoint => <<"localhost">>, - secret_access_key => <<"secret-access-key">>, - service => undefined}, - make_local_client(<<"access-key-id">>, - <<"secret-access-key">>, - <<"8000">>)). - -make_local_client_4_test() -> - ?assertEqual(#{access_key_id => <<"access-key-id">>, - port => <<"8000">>, - proto => <<"http">>, - region => <<"local">>, - endpoint => <<"endpoint">>, - secret_access_key => <<"secret-access-key">>, - service => undefined}, - make_local_client(<<"access-key-id">>, - <<"secret-access-key">>, - <<"8000">>, - <<"endpoint">>)). - --endif. diff --git a/src/aws_rds_iam_token.erl b/src/aws_rds_iam_token.erl deleted file mode 100644 index 644a2fd4..00000000 --- a/src/aws_rds_iam_token.erl +++ /dev/null @@ -1,67 +0,0 @@ --module(aws_rds_iam_token). --export([rds_token_create/4]). - --define(SIGNING_ID, <<"rds-db">>). --define(EMPTY_PAYLOAD_HASH, <<"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">>). - -%%==================================================================== -%% API -%%==================================================================== --spec rds_token_create(aws_client:aws_client(), binary(), non_neg_integer(), binary()) -> {ok, binary()}. -rds_token_create(Client, DBEndpoint, DBPort, DBUser) when is_map(Client) andalso - is_binary(DBEndpoint) andalso - is_integer(DBPort) andalso - is_binary(DBUser) -> - Method = <<"GET">>, - QueryParams = [{<<"Action">>, <<"connect">>}, {<<"DBUser">>, DBUser}], - Endpoint = <<"https://", DBEndpoint/binary, ":", (integer_to_binary(DBPort))/binary>>, - Url = aws_request:add_query(Endpoint, QueryParams), - AccessKeyID = aws_client:access_key_id(Client), - SecurityToken = aws_client:token(Client), - SecretAccessKey = aws_client:secret_access_key(Client), - Region = aws_client:region(Client), - Now = calendar:universal_time(), - Options0 = [ {ttl, timer:minutes(15) div 1000} %% Time in seconds - , {body_digest, ?EMPTY_PAYLOAD_HASH} - , {uri_encode_path, false} %% We already encode in build_path/4 - ], - Options = case SecurityToken of - undefined -> - Options0; - _ -> - [{session_token, hackney_url:urlencode(SecurityToken)} | Options0] - end, - <<"https://", SignedUrl/binary>> = aws_signature:sign_v4_query_params(AccessKeyID, SecretAccessKey, Region, ?SIGNING_ID, Now, Method, Url, Options), - {ok, SignedUrl}. - -%%==================================================================== -%% Unit tests -%%==================================================================== - --ifdef(TEST). - --include_lib("eunit/include/eunit.hrl"). --include_lib("hackney/include/hackney_lib.hrl"). - -fetch_auth_token_test() -> - Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"eu-west-1">>), - {ok, Url} = rds_token_create(Client, <<"db_endpoint">>, 5432, <<"db_user">>), - HackneyUrl = hackney_url:parse_url(Url), - ParsedQs = hackney_url:parse_qs(HackneyUrl#hackney_url.qs), - Credential = proplists:get_value(<<"X-Amz-Credential">>, ParsedQs), - [AccessKeyId, _ShortDate, Region, Service, Request] = binary:split(Credential, <<"/">>, [global]), - ?assertEqual(5432, HackneyUrl#hackney_url.port), - ?assertEqual("db_endpoint", HackneyUrl#hackney_url.host), - ?assertEqual(<<"">>, HackneyUrl#hackney_url.path), - ?assertEqual(9, length(ParsedQs)), - ?assertEqual(<<"AccessKeyID">>, AccessKeyId), - ?assertEqual(<<"eu-west-1">>, Region), - ?assertEqual(<<"rds-db">>, Service), - ?assertEqual(<<"aws4_request">>, Request), - ?assertEqual(<<"AWS4-HMAC-SHA256">>, proplists:get_value(<<"X-Amz-Algorithm">>, ParsedQs)), - ?assertEqual(<<"900">>, proplists:get_value(<<"X-Amz-Expires">>, ParsedQs)), - ?assertEqual(<<"Token">>, proplists:get_value(<<"X-Amz-Security-Token">>, ParsedQs)), - ?assertEqual(<<"host">>, proplists:get_value(<<"X-Amz-SignedHeaders">>, ParsedQs)). - --endif. diff --git a/src/aws_request.erl b/src/aws_request.erl deleted file mode 100644 index cfb3b744..00000000 --- a/src/aws_request.erl +++ /dev/null @@ -1,174 +0,0 @@ --module(aws_request). - --export([ add_headers/2 - , add_query/2 - , build_custom_headers/2 - , build_headers/2 - , method_to_binary/1 - , request/2 - , sign_request/5 - , sign_request/6 - ]). - --include_lib("hackney/include/hackney_lib.hrl"). - -%%==================================================================== -%% API -%%==================================================================== -%% Perform the actual request and depending on configuration, -%% retry if the response is off a retriable type. -request(RequestFun, Options) -> - RetryState = init_retry_state(proplists:get_value(retry_options, Options, undefined)), - do_request(RequestFun, RetryState). - -%% Generate headers with an AWS signature version 4 for the specified -%% request. -sign_request(Client, Method, URL, Headers0, Body) -> - sign_request(Client, Method, URL, Headers0, Body, [{uri_encode_path, false}]). - -sign_request(Client, Method, URL, Headers0, Body, Options) -> - AccessKeyID = aws_client:access_key_id(Client), - SecretAccessKey = aws_client:secret_access_key(Client), - Region = aws_client:region(Client), - Service = aws_client:service(Client), - Token = aws_client:token(Client), - Headers = case Token of - undefined -> Headers0; - _ -> [{<<"X-Amz-Security-Token">>, Token}|Headers0] - end, - aws_signature:sign_v4(AccessKeyID, SecretAccessKey, Region, Service, calendar:universal_time(), Method, URL, Headers, Body, Options). - -%% @doc Include additions only if they don't already exist in the provided list. -add_headers([], Headers) -> - Headers; -add_headers([{Name, _} = Header | Additions], Headers) -> - case lists:keyfind(Name, 1, Headers) of - false -> add_headers(Additions, [Header | Headers]); - _ -> add_headers(Additions, Headers) - end. - -%% @doc Build request headers based on a list key-value pairs -%% representing the mappings from param names to header names and a -%% map with the `params'. -build_headers(ParamsHeadersMapping, Params0) - when is_list(ParamsHeadersMapping), - is_map(Params0) -> - Fun = fun({HeaderName, ParamName}, {HeadersAcc, ParamsAcc}) -> - case maps:get(ParamName, ParamsAcc, undefined) of - undefined -> - {HeadersAcc, ParamsAcc}; - Value -> - Headers = [{HeaderName, Value} | HeadersAcc], - Params = maps:remove(ParamName, ParamsAcc), - {Headers, Params} - end - end, - lists:foldl(Fun, {[], Params0}, ParamsHeadersMapping). - -%% @doc Build custom request headers based on a list key-value pairs -%% representing the mappings from param names to header names and a -%% map with the `params'. -build_custom_headers(ParamsCustomHeadersMapping, Params0) - when is_list(ParamsCustomHeadersMapping), - is_map(Params0) -> - Fun = fun({HeaderName, ParamName}, {HeadersAcc, ParamsAcc}) -> - case maps:get(ParamName, ParamsAcc, undefined) of - undefined -> - {HeadersAcc, ParamsAcc}; - Value -> - Headers = [{<>, V} - || {K, V} <- maps:to_list(Value)] ++ HeadersAcc, - Params = maps:remove(ParamName, ParamsAcc), - {Headers, Params} - end - end, - lists:foldl(Fun, {[], Params0}, ParamsCustomHeadersMapping). - -%% @doc Add querystring to url is there are any parameters in the list --spec add_query(binary(), [{binary(), any()}]) -> binary(). -add_query(Url0, Query0) -> - HackneyUrl = hackney_url:parse_url(Url0), - NewQs = iolist_to_binary( - aws_util:encode_query( - hackney_url:parse_qs(HackneyUrl#hackney_url.qs) ++ Query0)), - HackneyUrlWithAddedQs = HackneyUrl#hackney_url{qs = NewQs}, - hackney_url:unparse_url(HackneyUrlWithAddedQs). - --spec method_to_binary(atom()) -> binary(). -method_to_binary(delete) -> <<"DELETE">>; -method_to_binary(get) -> <<"GET">>; -method_to_binary(head) -> <<"HEAD">>; -method_to_binary(options) -> <<"OPTIONS">>; -method_to_binary(patch) -> <<"PATCH">>; -method_to_binary(post) -> <<"POST">>; -method_to_binary(put) -> <<"PUT">>. - -%%==================================================================== -%% Internal functions -%%==================================================================== -do_request(RequestFun, RetryState) -> - Response = RequestFun(), - case classify_response(Response) of - retriable -> - case should_retry(RetryState) of - {ok, NewRetryState} -> - do_request(RequestFun, NewRetryState); - false -> - Response - end; - error -> - Response; - ok -> - Response - end. - -init_retry_state(undefined) -> - undefined; -init_retry_state({exponential_with_jitter, {MaxAttempts, BaseSleepTime, CapSleepTime}}) -> - #{ type => exponential_with_jitter - , n => 0 - , max_attempts => MaxAttempts - , base_sleep_time => BaseSleepTime - , cap_sleep_time => CapSleepTime - }. - -classify_response({error, _, {StatusCode, _, _}}) - when is_integer(StatusCode) andalso StatusCode >= 500 -> - retriable; -classify_response({error, {StatusCode, _}}) - when is_integer(StatusCode) andalso StatusCode >= 500 -> - retriable; -classify_response({error, _, {StatusCode, _, _}}) - when is_integer(StatusCode) -> - error; -classify_response({error, closed}) -> - retriable; -classify_response({error, connect_timeout}) -> - retriable; -classify_response({error, timeout}) -> - retriable; -classify_response({error, checkout_timeout}) -> - retriable; -classify_response({error, service_unavailable}) -> - retriable; -classify_response({error, _}) -> - error; -classify_response({ok, {_, _}}) -> - ok; -classify_response({ok, _, {_, _, _}}) -> - ok. - -should_retry(undefined) -> - false; -should_retry(#{ type := exponential_with_jitter - , n := N - , max_attempts := MaxAttempts}) when N =:= MaxAttempts -> - false; -should_retry(#{ type := exponential_with_jitter - , n := N - , base_sleep_time := BaseSleepTime - , cap_sleep_time := CapSleepTime} = RetryState0) -> - Temp = min(CapSleepTime, BaseSleepTime * trunc(math:pow(2, N))), - Sleep = Temp div 2 + rand:uniform(Temp div 2), - timer:sleep(Sleep), - {ok, RetryState0#{ n => N +1 }}. diff --git a/src/aws_s3_presigned_url.erl b/src/aws_s3_presigned_url.erl deleted file mode 100644 index 3bb793d9..00000000 --- a/src/aws_s3_presigned_url.erl +++ /dev/null @@ -1,236 +0,0 @@ -%%%% @doc -%%% -%%% Allows generating either a get or put presigned s3 url. -%%% This can be used by external clients such as cURL to access the object in question. -%%% -%%% See: -%%% - https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html -%%% - https://docs.aws.amazon.com/AmazonS3/latest/userguide/ShareObjectPreSignedURL.html --module(aws_s3_presigned_url). - --export([ make_presigned_v4_url/5, - make_presigned_v4_url/6, - make_presigned_v4_url/7 - ]). - --include_lib("hackney/include/hackney_lib.hrl"). - -%%==================================================================== -%% API -%%==================================================================== --spec make_presigned_v4_url(map(), get | put, integer(), binary(), binary()) -> {ok, binary()}. -make_presigned_v4_url(Client0, Method, ExpireSeconds, Bucket, Key) -> - make_presigned_v4_url(Client0, Method, ExpireSeconds, Bucket, Key, path). - --spec make_presigned_v4_url(map(), get | put, integer(), binary(), binary(),path|virtual_host) -> {ok, binary()}. -make_presigned_v4_url(Client0, Method, ExpireSeconds, Bucket, Key, Style) -> - make_presigned_v4_url(Client0, Method, ExpireSeconds, Bucket, Key, Style, undefined). - --spec make_presigned_v4_url(map(), get | put, integer(), binary(), binary(), path|virtual_host, undefined|binary()) -> - {ok, binary()}. -make_presigned_v4_url(Client0, Method, ExpireSeconds, Bucket, Key, Style, Tags) -> - MethodBin = aws_request:method_to_binary(Method), - Path = build_path(Client0,Bucket,Key,Style), - Client = Client0#{service => <<"s3">>}, - SecurityToken = aws_client:token(Client), - AccessKeyID = aws_client:access_key_id(Client), - SecretAccessKey = aws_client:secret_access_key(Client), - Region = aws_client:region(Client), - Service = aws_client:service(Client), - Host = build_host(<<"s3">>, Client, Bucket,Style), - URL = build_url(Host, Path, Client), - Now = calendar:universal_time(), - Options0 = [ {ttl, ExpireSeconds} - , {body_digest, <<"UNSIGNED-PAYLOAD">>} - , {uri_encode_path, false} %% We already encode in build_path/4 - ], - Options1 = - case SecurityToken of - undefined -> - Options0; - _ -> - [{session_token, hackney_url:urlencode(SecurityToken)} | Options0] - end, - Options = - case Tags of - undefined -> - Options1; - _ -> - [{tags, Tags} | Options1] - end, - {ok, aws_signature:sign_v4_query_params(AccessKeyID, SecretAccessKey, Region, Service, Now, MethodBin, URL, Options)}. - -%%==================================================================== -%% Internal functions -%%==================================================================== -%% Mocks are notoriously bad with host-style requests, just skip it and use path-style for anything local -%% At some points once the mocks catch up, we should remove this ugly hacks... -build_host(_EndpointPrefix, #{region := <<"local">>, endpoint := Endpoint}, _Bucket, _Style) -> - <>; -build_host(_EndpointPrefix, #{region := <<"local">>}, _Bucket, _Style) -> - <<"localhost">>; -build_host(EndpointPrefix, #{region := Region, endpoint := Endpoint}, _Bucket, path = _Style) -> - aws_util:binary_join([EndpointPrefix, Region, Endpoint], <<".">>); -build_host(EndpointPrefix, #{region := Region, endpoint := Endpoint}, Bucket, virtual_host = _Style) -> - aws_util:binary_join([Bucket, EndpointPrefix, Region, Endpoint], <<".">>). - -build_path(#{region := <<"local">>} = _Client,Bucket,Key, path = _Style) -> - ["/", aws_util:encode_uri(Bucket), "/", aws_util:encode_multi_segment_uri(Key), ""]; -build_path(_Client,Bucket,Key, path = _Style) -> - ["/", aws_util:encode_uri(Bucket), "/", aws_util:encode_multi_segment_uri(Key), ""]; -build_path(_Client,_Bucket,Key,virtual_host = _Style) -> - ["/", aws_util:encode_multi_segment_uri(Key), ""]. - -build_url(Host0, Path0, Client) -> - Proto = aws_client:proto(Client), - Path = erlang:iolist_to_binary(Path0), - Host = erlang:iolist_to_binary(Host0), - Port = aws_client:port(Client), - aws_util:binary_join([Proto, <<"://">>, Host, <<":">>, Port, Path], <<"">>). - -%%==================================================================== -%% Unit tests -%%==================================================================== - --ifdef(TEST). - --include_lib("eunit/include/eunit.hrl"). - -presigned_url_test() -> - Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"eu-west-1">>), - {ok, Url} = aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, <<"bucket">>, <<"key">>), - HackneyUrl = hackney_url:parse_url(Url), - ParsedQs = hackney_url:parse_qs(HackneyUrl#hackney_url.qs), - Credential = proplists:get_value(<<"X-Amz-Credential">>, ParsedQs), - [AccessKeyId, _ShortDate, Region, Service, Request] = binary:split(Credential, <<"/">>, [global]), - ?assertEqual(https, HackneyUrl#hackney_url.scheme), - ?assertEqual(443, HackneyUrl#hackney_url.port), - ?assertEqual("s3.eu-west-1.amazonaws.com", HackneyUrl#hackney_url.host), - ?assertEqual(<<"/bucket/key">>, HackneyUrl#hackney_url.path), - ?assertEqual(7, length(ParsedQs)), - ?assertEqual(<<"AccessKeyID">>, AccessKeyId), - ?assertEqual(<<"eu-west-1">>, Region), - ?assertEqual(<<"s3">>, Service), - ?assertEqual(<<"aws4_request">>, Request), - ?assertEqual(<<"AWS4-HMAC-SHA256">>, proplists:get_value(<<"X-Amz-Algorithm">>, ParsedQs)), - ?assertEqual(<<"3600">>, proplists:get_value(<<"X-Amz-Expires">>, ParsedQs)), - ?assertEqual(<<"Token">>, proplists:get_value(<<"X-Amz-Security-Token">>, ParsedQs)), - ?assertEqual(<<"host">>, proplists:get_value(<<"X-Amz-SignedHeaders">>, ParsedQs)). - -presigned_url_local_with_endpoint_test() -> - Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"local">>), - {ok, Url} = aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, <<"bucket">>, <<"key">>), - HackneyUrl = hackney_url:parse_url(Url), - ParsedQs = hackney_url:parse_qs(HackneyUrl#hackney_url.qs), - Credential = proplists:get_value(<<"X-Amz-Credential">>, ParsedQs), - [AccessKeyId, _ShortDate, Region, Service, Request] = binary:split(Credential, <<"/">>, [global]), - ?assertEqual(https, HackneyUrl#hackney_url.scheme), - ?assertEqual(443, HackneyUrl#hackney_url.port), - ?assertEqual("amazonaws.com", HackneyUrl#hackney_url.host), - ?assertEqual(<<"/bucket/key">>, HackneyUrl#hackney_url.path), - ?assertEqual(7, length(ParsedQs)), - ?assertEqual(<<"AccessKeyID">>, AccessKeyId), - ?assertEqual(<<"local">>, Region), - ?assertEqual(<<"s3">>, Service), - ?assertEqual(<<"aws4_request">>, Request), - ?assertEqual(<<"AWS4-HMAC-SHA256">>, proplists:get_value(<<"X-Amz-Algorithm">>, ParsedQs)), - ?assertEqual(<<"3600">>, proplists:get_value(<<"X-Amz-Expires">>, ParsedQs)), - ?assertEqual(<<"Token">>, proplists:get_value(<<"X-Amz-Security-Token">>, ParsedQs)), - ?assertEqual(<<"host">>, proplists:get_value(<<"X-Amz-SignedHeaders">>, ParsedQs)). - -presigned_url_local_without_endpoint_test() -> - Client0 = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"local">>), - Client = maps:without([endpoint],Client0), - {ok, Url} = aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, <<"bucket">>, <<"key">>), - HackneyUrl = hackney_url:parse_url(Url), - ParsedQs = hackney_url:parse_qs(HackneyUrl#hackney_url.qs), - Credential = proplists:get_value(<<"X-Amz-Credential">>, ParsedQs), - [AccessKeyId, _ShortDate, Region, Service, Request] = binary:split(Credential, <<"/">>, [global]), - ?assertEqual(https, HackneyUrl#hackney_url.scheme), - ?assertEqual(443, HackneyUrl#hackney_url.port), - ?assertEqual("localhost", HackneyUrl#hackney_url.host), - ?assertEqual(<<"/bucket/key">>, HackneyUrl#hackney_url.path), - ?assertEqual(7, length(ParsedQs)), - ?assertEqual(<<"AccessKeyID">>, AccessKeyId), - ?assertEqual(<<"local">>, Region), - ?assertEqual(<<"s3">>, Service), - ?assertEqual(<<"aws4_request">>, Request), - ?assertEqual(<<"AWS4-HMAC-SHA256">>, proplists:get_value(<<"X-Amz-Algorithm">>, ParsedQs)), - ?assertEqual(<<"3600">>, proplists:get_value(<<"X-Amz-Expires">>, ParsedQs)), - ?assertEqual(<<"Token">>, proplists:get_value(<<"X-Amz-Security-Token">>, ParsedQs)), - ?assertEqual(<<"host">>, proplists:get_value(<<"X-Amz-SignedHeaders">>, ParsedQs)). - -presigned_url_local_without_without_bucket_does_not_work_test() -> - Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"local">>), - ?assertException (error,function_clause,aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, undefined, <<"key">>)). - -presigned_url_path_style_test() -> - Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"eu-west-1">>), - {ok, Url} = aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, <<"bucket">>, <<"key">>,path), - HackneyUrl = hackney_url:parse_url(Url), - ParsedQs = hackney_url:parse_qs(HackneyUrl#hackney_url.qs), - Credential = proplists:get_value(<<"X-Amz-Credential">>, ParsedQs), - [AccessKeyId, _ShortDate, Region, Service, Request] = binary:split(Credential, <<"/">>, [global]), - ?assertEqual(https, HackneyUrl#hackney_url.scheme), - ?assertEqual(443, HackneyUrl#hackney_url.port), - ?assertEqual("s3.eu-west-1.amazonaws.com", HackneyUrl#hackney_url.host), - ?assertEqual(<<"/bucket/key">>, HackneyUrl#hackney_url.path), - ?assertEqual(7, length(ParsedQs)), - ?assertEqual(<<"AccessKeyID">>, AccessKeyId), - ?assertEqual(<<"eu-west-1">>, Region), - ?assertEqual(<<"s3">>, Service), - ?assertEqual(<<"aws4_request">>, Request), - ?assertEqual(<<"AWS4-HMAC-SHA256">>, proplists:get_value(<<"X-Amz-Algorithm">>, ParsedQs)), - ?assertEqual(<<"3600">>, proplists:get_value(<<"X-Amz-Expires">>, ParsedQs)), - ?assertEqual(<<"Token">>, proplists:get_value(<<"X-Amz-Security-Token">>, ParsedQs)), - ?assertEqual(<<"host">>, proplists:get_value(<<"X-Amz-SignedHeaders">>, ParsedQs)). - -presigned_url_virtual_host_style_test() -> - Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"eu-west-1">>), - {ok, Url} = aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, <<"bucket">>, <<"key">>,virtual_host), - HackneyUrl = hackney_url:parse_url(Url), - ParsedQs = hackney_url:parse_qs(HackneyUrl#hackney_url.qs), - Credential = proplists:get_value(<<"X-Amz-Credential">>, ParsedQs), - [AccessKeyId, _ShortDate, Region, Service, Request] = binary:split(Credential, <<"/">>, [global]), - ?assertEqual(https, HackneyUrl#hackney_url.scheme), - ?assertEqual(443, HackneyUrl#hackney_url.port), - ?assertEqual("bucket.s3.eu-west-1.amazonaws.com", HackneyUrl#hackney_url.host), - ?assertEqual(<<"/key">>, HackneyUrl#hackney_url.path), - ?assertEqual(7, length(ParsedQs)), - ?assertEqual(<<"AccessKeyID">>, AccessKeyId), - ?assertEqual(<<"eu-west-1">>, Region), - ?assertEqual(<<"s3">>, Service), - ?assertEqual(<<"aws4_request">>, Request), - ?assertEqual(<<"AWS4-HMAC-SHA256">>, proplists:get_value(<<"X-Amz-Algorithm">>, ParsedQs)), - ?assertEqual(<<"3600">>, proplists:get_value(<<"X-Amz-Expires">>, ParsedQs)), - ?assertEqual(<<"Token">>, proplists:get_value(<<"X-Amz-Security-Token">>, ParsedQs)), - ?assertEqual(<<"host">>, proplists:get_value(<<"X-Amz-SignedHeaders">>, ParsedQs)). - -presigned_url_tags_test() -> - Client = aws_client:make_temporary_client(<<"AccessKeyID">>, <<"SecretAccessKey">>, - <<"Token">>, <<"eu-west-1">>), - {ok, Url} = aws_s3_presigned_url:make_presigned_v4_url(Client, put, 3600, <<"bucket">>, <<"key">>, path, <<"key1=value1&key2=value2">>), - HackneyUrl = hackney_url:parse_url(Url), - ParsedQs = hackney_url:parse_qs(HackneyUrl#hackney_url.qs), - Credential = proplists:get_value(<<"X-Amz-Credential">>, ParsedQs), - [AccessKeyId, _ShortDate, Region, Service, Request] = binary:split(Credential, <<"/">>, [global]), - ?assertEqual(https, HackneyUrl#hackney_url.scheme), - ?assertEqual(443, HackneyUrl#hackney_url.port), - ?assertEqual("s3.eu-west-1.amazonaws.com", HackneyUrl#hackney_url.host), - ?assertEqual(<<"/bucket/key">>, HackneyUrl#hackney_url.path), - ?assertEqual(7, length(ParsedQs)), - ?assertEqual(<<"AccessKeyID">>, AccessKeyId), - ?assertEqual(<<"eu-west-1">>, Region), - ?assertEqual(<<"s3">>, Service), - ?assertEqual(<<"aws4_request">>, Request), - ?assertEqual(<<"AWS4-HMAC-SHA256">>, proplists:get_value(<<"X-Amz-Algorithm">>, ParsedQs)), - ?assertEqual(<<"3600">>, proplists:get_value(<<"X-Amz-Expires">>, ParsedQs)), - ?assertEqual(<<"Token">>, proplists:get_value(<<"X-Amz-Security-Token">>, ParsedQs)), - ?assertEqual(<<"host;x-amz-tagging">>, proplists:get_value(<<"X-Amz-SignedHeaders">>, ParsedQs)). --endif. diff --git a/src/aws_util.erl b/src/aws_util.erl deleted file mode 100644 index 42d54c56..00000000 --- a/src/aws_util.erl +++ /dev/null @@ -1,383 +0,0 @@ --module(aws_util). - --export([base16/1, - binary_join/2, - hmac_sha256/2, - hmac_sha256_hexdigest/2, - sha256_hexdigest/1, - encode_query/1, - encode_uri/1, - encode_multi_segment_uri/1, - encode_xml/1, - decode_xml/1, - get_in/2, - get_in/3 - ]). - --include_lib("xmerl/include/xmerl.hrl"). - -%%==================================================================== -%% API -%%==================================================================== - -%% @doc Base16 encode binary data. -base16(Data) -> - << <<(hex(N div 16)), (hex(N rem 16))>> || <> <= Data >>. - -%% @doc Join binary values using the specified separator. -binary_join([], _) -> <<"">>; -binary_join([H|[]], _) -> H; -binary_join(L, Sep) when is_list(Sep) -> - binary_join(L, list_to_binary(Sep)); -binary_join([H|T], Sep) -> - binary_join(T, H, Sep). - -%% @doc Create an HMAC-SHA256 hexdigest for Key and Message. -hmac_sha256_hexdigest(Key, Message) -> - aws_util:base16(hmac_sha256(Key, Message)). - -%% @doc Create an HMAC-SHA256 hexdigest for Key and Message. -hmac_sha256(Key, Message) -> - crypto_hmac(sha256, Key, Message). - -%% @doc Create a SHA256 hexdigest for Value. -sha256_hexdigest(Value) -> - aws_util:base16(crypto:hash(sha256, Value)). - -%% @doc Encode URI taking into account if it contains more than one -%% segment. -encode_multi_segment_uri(Value) -> - Encoded = [ encode_uri(Segment) - || Segment <- binary:split(Value, <<"/">>, [global]) - ], - binary_join(Encoded, <<"/">>). - -%% @doc Encode URI into a percent-encoding string. -encode_uri(Value) when is_list(Value) -> - list_to_binary(Value); -encode_uri(Value) when is_binary(Value) -> - << (uri_encode_path_byte(Byte)) || <> <= Value >>. - --spec uri_encode_path_byte(byte()) -> binary(). -uri_encode_path_byte($/) -> <<"/">>; -uri_encode_path_byte(Byte) - when $0 =< Byte, Byte =< $9; - $a =< Byte, Byte =< $z; - $A =< Byte, Byte =< $Z; - Byte =:= $~; - Byte =:= $_; - Byte =:= $-; - Byte =:= $. -> - <>; -uri_encode_path_byte(Byte) -> - H = Byte band 16#F0 bsr 4, - L = Byte band 16#0F, - <<"%", (hex(H, upper)), (hex(L, upper))>>. - -%% @doc Encode the map's key/value pairs as a querystring. -%% The query string must be sorted. -%% The query string for query params that do not contain a value such as "key" -%% should be encoded as "key=". -%% Without this fix, the request will result in a SignatureDoesNotMatch error. -encode_query(QueryL) when is_list(QueryL) -> - uri_string:compose_query( - lists:sort( - lists:map(fun({K, true}) -> {K, ""}; - ({K, V}) when is_binary(V) -> {K, V}; - ({K, V}) when is_float(V) -> {K, float_to_binary(V)}; - ({K, V}) when is_integer(V) -> {K, integer_to_binary(V)} - end, QueryL))); -encode_query(Map) when is_map(Map) -> - encode_query(maps:to_list(Map)). - -%% @doc Encode an Erlang map as XML -%% -%% All keys must be binaries. Values can be a binary, a list, an -%% integer a float or another nested map. -encode_xml(Map) -> - Result = lists:map(fun encode_xml_key_value/1, maps:to_list(Map)), - iolist_to_binary(Result). - -%% @doc Decode XML into a map representation -%% -%% When there is more than one element with the same tag name, their -%% values get merged into a list. -%% -%% If the content is only text then a key with the element name and a -%% value with the content is inserted. -%% -%% If the content is a mix between text and child elements, then the -%% elements are processed as described above and all the text parts -%% are merged under the binary `__text' key. -decode_xml(Xml) -> - %% See: https://elixirforum.com/t/utf-8-issue-with-erlang-xmerl-scan-function/1668/9 - XmlString = erlang:binary_to_list(Xml), - Opts = [{hook_fun, fun hook_fun/2}], - {Element, []} = xmerl_scan:string(XmlString, Opts), - Element. - -%% @doc Get a value from nested maps --spec get_in([any()], any()) -> any(). -get_in(Keys, V) -> - get_in(Keys, V, undefined). - -%% @doc Get a value from nested maps, return default value if missing --spec get_in([any()], any(), any()) -> any(). -get_in([], V, _Default) -> - V; -get_in([K | Keys], Map, Default) when is_map(Map) -> - case maps:find(K, Map) of - {ok, V} -> get_in(Keys, V, Default); - error -> Default - end. - -%%==================================================================== -%% Internal functions -%%==================================================================== - --spec encode_xml_key_value({binary(), any()}) -> iolist(). -encode_xml_key_value({K, V}) when is_binary(K), is_binary(V) -> - ["<", K, ">", V, ""]; -encode_xml_key_value({K, Values}) when is_binary(K), is_list(Values) -> - [encode_xml_key_value({K, V}) || V <- Values]; -encode_xml_key_value({K, V}) when is_binary(K), is_integer(V) -> - ["<", K, ">", integer_to_binary(V), ""]; -encode_xml_key_value({K, V}) when is_binary(K), is_float(V) -> - ["<", K, ">", float_to_binary(V), ""]; -encode_xml_key_value({K, V}) when is_binary(K), is_map(V) -> - [ "<", K, ">" - , lists:map(fun encode_xml_key_value/1, maps:to_list(V)) - , "" - ]. - --define(TEXT, <<"__text">>). - -%% @doc Callback hook_fun for xmerl parser -hook_fun(#xmlElement{name = Tag, content = Content} , GlobalState) -> - Value = case lists:foldr(fun content_to_map/2, none, Content) of - V = #{?TEXT := Text} -> - case string:trim(Text) of - <<>> -> maps:remove(?TEXT, V); - Trimmed -> V#{?TEXT => Trimmed} - end; - V -> V - end, - {#{atom_to_binary(Tag, utf8) => Value}, GlobalState}; -hook_fun(#xmlText{value = Text}, GlobalState) -> - {unicode:characters_to_binary(Text), GlobalState}. - -%% @doc Convert the content of an Xml node into a map. -content_to_map(X, none) -> - X; -content_to_map(X, Acc) when is_map(X), is_map(Acc) -> - [{Tag, Value}] = maps:to_list(X), - case maps:is_key(Tag, Acc) of - true -> - UpdateFun = fun(L) when is_list(L) -> - [Value | L]; - (V) -> [Value, V] - end, - maps:update_with(Tag, UpdateFun, Acc); - false -> maps:merge(Acc, X) - end; -content_to_map(X, #{?TEXT := Text} = Acc) - when is_binary(X), is_map(Acc) -> - Acc#{?TEXT => <>}; -content_to_map(X, Acc) when is_binary(X), is_map(Acc) -> - Acc#{?TEXT => X}; -content_to_map(X, Acc) when is_binary(X), is_binary(Acc) -> - <>; -content_to_map(X, Acc) when is_map(X), is_binary(Acc) -> - X#{?TEXT => Acc}. - -%% @doc Convert an integer in the 0-16 range to a hexadecimal byte -%% representation. -hex(N) -> - hex(N, lower). - -hex(N, upper) -> - hex(N, $A); -hex(N, lower) -> - hex(N, $a); -hex(N, _Char) when N >= 0, N < 10 -> - N + $0; -hex(N, Char) when N < 16 -> - N - 10 + Char. -binary_join([], Acc, _) -> - Acc; -binary_join([H|T], Acc, Sep) -> - binary_join(T, <>, Sep). - -%% this can be simplified if we drop support for OTP < 21 -%% this can be removed if we drop support for OTP < 23 --ifdef(OTP_RELEASE). % OTP >= 21 --if(?OTP_RELEASE >= 23). --define(USE_CRYPTO_MAC_4, true). --else. --undef(USE_CRYPTO_MAC_4). --endif. --else. % OTP < 21 --undef(USE_CRYPTO_MAC_4). --endif. - --ifdef(USE_CRYPTO_MAC_4). -crypto_hmac(Sha, Key, Data) -> crypto:mac(hmac, Sha, Key, Data). --else. -crypto_hmac(Sha, Key, Data) -> crypto:hmac(Sha, Key, Data). --endif. - -%%==================================================================== -%% Unit tests -%%==================================================================== - --ifdef(TEST). --include_lib("eunit/include/eunit.hrl"). - -%% binary_join/2 joins a list of binary values, separated by a separator -%% character, into a single binary value. -binary_join_test() -> - ?assertEqual(binary_join([<<"a">>, <<"b">>, <<"c">>], <<",">>), - <<"a,b,c">>). - -%% binary_join/2 correctly joins binary values with a multi-character -%% separator. -binary_join_with_multi_character_separator_test() -> - ?assertEqual(binary_join([<<"a">>, <<"b">>, <<"c">>], <<", ">>), - <<"a, b, c">>). - -%% binary_join/2 converts a list containing a single binary into the binary -%% itself. -binary_join_with_single_element_list_test() -> - ?assertEqual(binary_join([<<"a">>], <<",">>), <<"a">>). - -%% binary_join/2 returns an empty binary value when an empty list is -%% provided. -binary_join_with_empty_list_test() -> - ?assertEqual(binary_join([], <<",">>), <<"">>). - -%% sha256_hexdigest/1 returns a SHA256 hexdigest for an empty value. -sha256_hexdigest_with_empty_value_test() -> - ?assertEqual( - <<"e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855">>, - sha256_hexdigest(<<"">>)). - -%% sha256_hexdigest/1 returns a SHA256 hexdigest for a non-empty body. -sha256_hexdigest_test() -> - ?assertEqual( - <<"315f5bdb76d078c43b8ac0064e4a0164612b1fce77c869345bfc94c75894edd3">>, - sha256_hexdigest(<<"Hello, world!">>)). - -%% hmac_sha256/2 returns a SHA256 HMAC for a message. -hmac_sha256_test() -> - ?assertEqual( - <<110, 158, 242, 155, 117, 255, 252, 91, - 122, 186, 229, 39, 213, 143, 218, 219, - 47, 228, 46, 114, 25, 1, 25, 118, - 145, 115, 67, 6, 95, 88, 237, 74>>, - hmac_sha256(<<"key">>, <<"message">>)). - -%% hmac_sha256_hexdigest/2 returns an HMAC SHA256 hexdigest for a message. -hmac_sha256_hexdigest_test() -> - ?assertEqual( - <<"6e9ef29b75fffc5b7abae527d58fdadb2fe42e7219011976917343065f58ed4a">>, - hmac_sha256_hexdigest(<<"key">>, <<"message">>)). - -%% decode_xml handles lists correctly by merging values in a list. -decode_xml_lists_test() -> - ?assertEqual( - #{ <<"person">> => - #{ <<"name">> => <<"foo">> - , <<"addresses">> => #{<<"address">> => [<<"1">>, <<"2">>]} - } - }, - decode_xml(<<"" - " foo" - " " - "
1
" - "
2
" - " " - "">>)). - -%% decode_xml handles multiple text elments mixed with other elements correctly. -decode_xml_text_test() -> - ?assertEqual( #{ <<"person">> => - #{ <<"name">> => <<"foo">> - , ?TEXT => <<"random">> - } - } - , decode_xml(<<"" - " foo" - " random" - "">>) - ), - - ?assertEqual( #{<<"person">> => #{ <<"name">> => <<"foo">> - , <<"age">> => <<"42">> - , ?TEXT => <<"random text">> - } - } - , decode_xml(<<"" - " foo" - " random" - " 42" - " text" - "">>) - ). - -decode_utf8_xml_text_test() -> - ?assertEqual( #{ <<"person">> => - #{ <<"name">> => <<"сергей"/utf8>> - , ?TEXT => <<"random">> - } - } - , decode_xml(<<"" - " сергей" - " random" - ""/utf8>>) - ). - - -%% get_in fetches the correct values and does fail when the path doesn't exist -get_in_test() -> - Map = #{ <<"person">> => - #{ <<"error">> => #{ <<"code">> => <<"Code">> - , <<"message">> => <<"Message">> - } - } - }, - CodePath = [<<"person">>, <<"error">>, <<"code">>], - MessagePath = [<<"person">>, <<"error">>, <<"message">>], - FooPath = [<<"person">>, <<"error">>, <<"foo">>], - ?assertEqual(<<"Code">>, get_in(CodePath, Map)), - ?assertEqual(<<"Message">>, get_in(MessagePath, Map)), - ?assertEqual(undefined, get_in(FooPath, Map)), - ?assertEqual(default, get_in(FooPath, Map, default)). - -%% encode_uri correctly encode segment of an URI -encode_uri_test() -> - Segment = <<"hello world!">>, - ?assertEqual(<<"hello%20world%21">>, encode_uri(Segment)). - -encode_uri_parenthesis_test() -> - Segment = <<"hello world(!)">>, - ?assertEqual(<<"hello%20world%28%21%29">>, encode_uri(Segment)). - -encode_uri_special_chars_test() -> - Segment = <<"file_!-_.(*)&=;:+ ,?{^}%]>[~<#`|.content">>, - ?assertEqual(<<"file_%21-_.%28%2A%29%26%3D%3B%3A%2B%20%2C%3F%7B%5E%7D%25%5D%3E%5B~%3C%23%60%7C.content">>, - encode_uri(Segment)). - -%% encode_multi_segment_uri correctly encode each segment of an URI -encode_multi_segment_uri_test() -> - MultiSegment = <<"hello /world!">>, - ?assertEqual(<<"hello%20/world%21">>, encode_multi_segment_uri(MultiSegment)). - -encode_query_test() -> - Query = [{<<"two">>, <<"2">>}], - ?assertEqual(<<"two=2">>, encode_query(Query)). - -encode_query_sorted_test() -> - Query = [{<<"two">>, <<"2">>}, {<<"one">>, <<"1">>}], - ?assertEqual(<<"one=1&two=2">>, encode_query(Query)). - --endif.