Issue #103: Migrate from aws-sdk-go to aws-sdk-go-v2

Goal here has been to minimize the diff in aws-elixir and aws-erlang to a bare minimum.

Some minor diffs were unavoidable and deemed acceptable
* Fix SuccessStatusCode to always be 200 if not provided by AWS in the API
  * Before SuccessStatusCode was set to undefined/nil. This was unhandled and would've caused errors.
* Fix aws-erlang/issues/148 to allow send_body_as_binary & receive_body_as_binary to be sent in as options
  * The default remains the same, but certain APIs may require a change of the default depending on the in- or output
* Drop abbreviation from metadata. It is no longer available in aws-sdk-go-v2 and unused in aws-elixir
* Slight formatting changes on aws-erlang docs as the new docs contain a lot more newlines and hence cause more noise
* Uri Paths may have changed. This should not impact the clients. It was likely broken in the aws-sdk-go
* QueryMap may have changed but should not impact the client. It was likely broken in the aws-sdk-go
* Docs have changed (massively) since almost all documentation has been updated

Co-authored-by: Amin Arria <[email protected]>
Co-authored-by: Gustavo Mora González <[email protected]>
Co-authored-by: Philip Sampaio <[email protected]>

.github/workflows/gen_elixir.yml

@@ -3,7 +3,7 @@ name: Generate aws-elixir
 on:
   push:
     branches:
-      - master 
+      - master
   pull_request:
   workflow_dispatch:
 
@@ -29,19 +29,19 @@ jobs:
           sudo apt-get update
           DEBIAN_FRONTEND=noninteractive sudo apt-get -y --no-install-recommends install build-essential cmake
           mix deps.get
-      - name: Checkout aws/aws-sdk-go
+      - name: Checkout aws/aws-sdk-go-v2
         uses: actions/checkout@v2
         with:
-          repository: aws/aws-sdk-go
-          path: aws/aws-sdk-go/
+          repository: aws/aws-sdk-go-v2
+          path: aws/aws-sdk-go-v2/
       - name: Checkout aws-elixir
         uses: actions/checkout@v2
         with:
           repository: aws-beam/aws-elixir
           path: aws-beam/aws-elixir
       - name: Generate aws-elixir
         env:
-          SPEC_PATH: aws/aws-sdk-go/models/apis
+          SPEC_PATH: aws/aws-sdk-go-v2/codegen/sdk-codegen/aws-models
           TEMPLATE_PATH: priv
           ELIXIR_OUTPUT_PATH: aws-beam/aws-elixir/lib/aws/generated
         run: |

.github/workflows/gen_erlang.yml

@@ -9,45 +9,60 @@ on:
 
 jobs:
   build:
-    runs-on: ubuntu-latest
-    name: Elixir ${{ matrix.elixir }} / OTP ${{ matrix.erlang }}
-
     strategy:
-      fail-fast: false
       matrix:
+        platform: [ubuntu-latest]
         elixir: ["1.13.4"]
         erlang: ["25.0.4"]
         rebar3: ['3.20.0']
-
+    runs-on: ${{ matrix.platform }}
+    services:
+      ddb:
+        image: amazon/dynamodb-local:1.21.0
+        ports:
+          - 8000:8000
+      s3mock:
+        image: adobe/s3mock:2.11.0
+        ports:
+          - 9090:9090
     steps:
-      - uses: actions/checkout@v2
-      - uses: erlef/setup-beam@v1
+      - uses: actions/checkout@v4
+      - name: Set up Erlang / Elixir / Rebar3
+        uses: erlef/setup-beam@v1
         with:
-          otp-version: "${{ matrix.erlang }}"
-          elixir-version: "${{ matrix.elixir }}"
-          rebar3-version: "${{ matrix.rebar3 }}"
-      - name: Install Dependencies
-        run: |
-          sudo apt-get update
-          DEBIAN_FRONTEND=noninteractive sudo apt-get -y --no-install-recommends install build-essential cmake
-          mix deps.get
-      - name: Checkout aws/aws-sdk-go
-        uses: actions/checkout@v2
+          otp-version: ${{matrix.erlang}}
+          elixir-version: ${{matrix.elixir}}
+          rebar3-version: ${{matrix.rebar3}}
+      - name: Checkout aws/aws-sdk-go-v2
+        uses: actions/checkout@v4
         with:
-          repository: aws/aws-sdk-go
-          path: aws/aws-sdk-go/
+          repository: aws/aws-sdk-go-v2
+          path: aws/aws-sdk-go-v2/
       - name: Checkout aws-erlang
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           repository: aws-beam/aws-erlang
           path: aws-beam/aws-erlang
+      - name: Get mix dependencies
+        run: mix deps.get
       - name: Generate aws-erlang
         env:
-          SPEC_PATH: aws/aws-sdk-go/models/apis
+          SPEC_PATH: aws/aws-sdk-go-v2/codegen/sdk-codegen/aws-models
           TEMPLATE_PATH: priv
           ERLANG_OUTPUT_PATH: aws-beam/aws-erlang/src
         run: mix run generate.exs erlang $SPEC_PATH $TEMPLATE_PATH $ERLANG_OUTPUT_PATH
-      - name: Run Erlang tests
+      - name: Compile
+        run: rebar3 compile
+        working-directory: aws-beam/aws-erlang
+      - name: Run EUnit Tests
         run: rebar3 eunit
         working-directory: aws-beam/aws-erlang
-
+      - name: Run Common Tests
+        run: rebar3 ct
+        working-directory: aws-beam/aws-erlang
+        env:
+          DYNAMODB_HOST: localhost
+          S3MOCK_HOST: localhost
+      - name: Produce Documentation
+        run: rebar3 ex_doc
+        working-directory: aws-beam/aws-erlang

lib/aws_codegen.ex

@@ -18,8 +18,7 @@ defmodule AWS.CodeGen do
   defmodule Service do
     @moduledoc false
 
-    defstruct abbreviation: nil,
-              actions: [],
+    defstruct actions: [],
               api_version: nil,
               credential_scope: nil,
               content_type: nil,
@@ -141,16 +140,16 @@ defmodule AWS.CodeGen do
 
   @spec api_specs(binary(), :elixir | :erlang) :: [Spec.t()]
   defp api_specs(base_path, language) do
-    search_path = Path.join(base_path, "*/*")
+    search_path = Path.join(base_path, "*")
     IO.puts("Parsing specs in #{search_path}")
 
-    for path <- Path.wildcard(search_path) do
-      Spec.parse(path, language)
+    for file <- Path.wildcard(search_path) do
+      Spec.parse(file, language)
     end
   end
 
   defp get_endpoints_spec(base_path) do
-    Path.join([base_path, "..", "endpoints", "endpoints.json"])
+    Path.join([base_path, "../../", "smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen", "endpoints.json"])
     |> Spec.parse_json()
     |> get_in(["partitions"])
     |> Enum.filter(fn x -> x["partition"] == "aws" end)

lib/aws_codegen/docstring.ex

@@ -272,7 +272,7 @@ defmodule AWS.CodeGen.Docstring do
         text = Floki.text(children)
 
         if String.contains?(text, "\n") do
-          "\n```\n#{String.trim_leading(text, "\n")}'''#{@two_break_lines}"
+          "\n```\n#{String.replace(text, "\n", "")}'''"
         else
           ## ex_doc blows up on these sorts of things as it sees them as a reference to a function.
           ## Just ignore them as they refer to aws-sdk-go based implementations and we don't really care about that
@@ -297,9 +297,9 @@ defmodule AWS.CodeGen.Docstring do
             "`#{Floki.text(children)}'"
         end
 
-      other ->
-        other
-    end)
+      {_, _attrs, children} ->
+        "#{Floki.text(children)}"
+      end)
     |> Floki.raw_html(encode: true)
   end
 

lib/aws_codegen/name.ex

@@ -12,6 +12,7 @@ defmodule AWS.CodeGen.Name do
   """
   def to_snake_case(text) do
     text
+    |> String.replace(~r/com\.amazonaws\.[^#]+#/, "")
     |> String.replace(@names_to_capitalize, &String.capitalize(&1))
     |> String.to_charlist()
     |> Enum.map_join(&char_to_snake_case/1)

lib/aws_codegen/post_service.ex

@@ -53,65 +53,94 @@ defmodule AWS.CodeGen.PostService do
   `:elixir` or `:erlang`.
   """
   def load_context(language, %AWS.CodeGen.Spec{} = spec, endpoints_spec) do
-    metadata = spec.api["metadata"]
-    actions = collect_actions(language, spec.api, spec.doc)
-
-    endpoint_prefix = metadata["endpointPrefix"]
+    %Service{actions: []}
+    service = spec.api["shapes"][spec.shape_name]
+    traits = service["traits"]
+    actions = collect_actions(language, spec.api)
+    endpoint_prefix = traits["aws.api#service"]["endpointPrefix"] || traits["aws.api#service"]["arnNamespace"]
     endpoint_info = endpoints_spec["services"][endpoint_prefix]
     is_global = not is_nil(endpoint_info) and not Map.get(endpoint_info, "isRegionalized", true)
-
     credential_scope =
       if is_global do
         endpoint_info["endpoints"]["aws-global"]["credentialScope"]["region"]
       end
-
-    json_version = metadata["jsonVersion"]
-    protocol = metadata["protocol"]
+    json_version = AWS.CodeGen.Util.get_json_version(service)
+    protocol = spec.protocol |> to_string()
     content_type = @configuration[protocol][:content_type]
     content_type = content_type <> if protocol == "json", do: json_version, else: ""
-
     signing_name =
-      case metadata["signingName"] do
-        nil -> endpoint_prefix
-        sn -> sn
+      if String.starts_with?(endpoint_prefix, "api.") do
+        String.replace(endpoint_prefix, "api.", "")
+      else
+        endpoint_prefix
       end
 
     %Service{
-      abbreviation: metadata["serviceAbbreviation"],
       actions: actions,
-      api_version: metadata["apiVersion"],
+      api_version: service["version"],
       credential_scope: credential_scope,
       content_type: content_type,
-      docstring: Docstring.format(language, spec.doc["service"]),
+      docstring: Docstring.format(language, AWS.CodeGen.Util.service_docs(service)),
       decode: Map.fetch!(@configuration[protocol][language], :decode),
       encode: Map.fetch!(@configuration[protocol][language], :encode),
       endpoint_prefix: endpoint_prefix,
       is_global: is_global,
       json_version: json_version,
       language: language,
       module_name: spec.module_name,
-      protocol: protocol,
+      protocol: protocol |> to_string() |> String.replace("_", "-"),
       signing_name: signing_name,
-      signature_version: metadata["signatureVersion"],
-      service_id: metadata["serviceId"],
-      target_prefix: metadata["targetPrefix"]
+      signature_version: AWS.CodeGen.Util.get_signature_version(service),
+      service_id: AWS.CodeGen.Util.get_service_id(service),
+      target_prefix: target_prefix(spec.api)
     }
   end
 
-  defp collect_actions(language, api_spec, doc_spec) do
-    Enum.map(api_spec["operations"], fn {operation, metadata} ->
+  defp target_prefix(api) do
+    api["shapes"]
+    |> Enum.find(fn {_key, value} -> match?(%{"type" => "service"}, value) end)
+    |> case do
+      {key, _} ->
+        String.replace(key, ~r/.*#/, "")
+      nil -> nil
+    end
+  end
+
+  defp collect_actions(language, api_spec) do
+    shapes = api_spec["shapes"]
+
+    operations =
+      Enum.reduce(shapes, [], fn {_, shape}, acc ->
+        case shape["type"] do
+          "service" ->
+
+            [acc | List.wrap(shape["operations"])]
+          "resource" ->
+            [shape["operations"], shape["collectionOperations"], shape["create"], shape["put"], shape["read"], shape["update"], shape["delete"], shape["list"]]
+            |> Enum.reject(&is_nil/1)
+            |> Kernel.++(acc)
+          _ ->
+            acc
+        end
+      end)
+      |> List.flatten()
+      |> Enum.map(fn %{"target" => target} -> target end)
+
+    Enum.map(operations, fn operation ->
+      operation_spec = shapes[operation]
       %Action{
         arity: 3,
         docstring:
           Docstring.format(
             language,
-            doc_spec["operations"][operation]
+            operation_spec["traits"]["smithy.api#documentation"]
           ),
         function_name: AWS.CodeGen.Name.to_snake_case(operation),
-        host_prefix: get_in(metadata, ["endpoint", "hostPrefix"]),
-        name: operation
+        host_prefix: operation_spec["traits"]["smithy.api#endpoint"]["hostPrefix"],
+        name: String.replace(operation, ~r/com\.amazonaws\.[^#]+#/, "")
       }
     end)
     |> Enum.sort(fn a, b -> a.function_name < b.function_name end)
+    |> Enum.uniq()
   end
 end