Merge pull request #109 from aws-beam/generate-types-and-type-specs-f…

…or-all-generated-code

Generate types and type specs for all generated functions in aws-elixir and aws-erlang

lib/aws_codegen.ex

@@ -33,6 +33,7 @@ defmodule AWS.CodeGen do
               protocol: nil,
               signature_version: nil,
               service_id: nil,
+              shapes: %{},
               signing_name: nil,
               target_prefix: nil
   end
@@ -96,7 +97,7 @@ defmodule AWS.CodeGen do
         end
       )
 
-    Enum.each(tasks, fn task -> Task.await(task, 60_000) end)
+    Enum.each(tasks, fn task -> Task.await(task, 120_000) end)
   end
 
   defp generate_code(spec, language, endpoints_spec, template_base_path, output_path) do
@@ -107,15 +108,17 @@ defmodule AWS.CodeGen do
       template_path = Path.join(template_base_path, template)
 
       context = protocol_service.load_context(language, spec, endpoints_spec)
+
       case Map.get(context, :actions) do
         [] ->
           IO.puts(["Skipping ", spec.module_name, " due to no actions"])
+
         _ ->
           code = render(context, template_path)
 
-      IO.puts(["Writing ", spec.module_name, " to ", output_path])
+          IO.puts(["Writing ", spec.module_name, " to ", output_path])
 
-      File.write(output_path, code)
+          File.write(output_path, code)
       end
     else
       IO.puts("Failed to generate #{spec.module_name}, protocol #{spec.protocol}")
@@ -149,7 +152,12 @@ defmodule AWS.CodeGen do
   end
 
   defp get_endpoints_spec(base_path) do
-    Path.join([base_path, "../../", "smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen", "endpoints.json"])
+    Path.join([
+      base_path,
+      "../../",
+      "smithy-aws-go-codegen/src/main/resources/software/amazon/smithy/aws/go/codegen",
+      "endpoints.json"
+    ])
     |> Spec.parse_json()
     |> get_in(["partitions"])
     |> Enum.filter(fn x -> x["partition"] == "aws" end)

lib/aws_codegen/docstring.ex

@@ -39,8 +39,10 @@ defmodule AWS.CodeGen.Docstring do
     |> String.trim_trailing()
     |> String.replace(@two_break_lines, "\n%%\n")
     |> String.replace(~r/'/, "'")
-    |> String.replace("`AVAILABLE`", "`AVAILABLE'") # aws-sdk-go docs are broken for this, hack it to make the edocs work
-    |> String.replace("`PENDING`", "`PENDING'") # aws-sdk-go docs are broken for this, hack it to make the edocs work
+    # aws-sdk-go docs are broken for this, hack it to make the edocs work
+    |> String.replace("`AVAILABLE`", "`AVAILABLE'")
+    # aws-sdk-go docs are broken for this, hack it to make the edocs work
+    |> String.replace("`PENDING`", "`PENDING'")
   end
 
   defp split_first_sentence_in_one_line(doc) do
@@ -292,6 +294,7 @@ defmodule AWS.CodeGen.Docstring do
         case Enum.find(attrs, fn {attr, _} -> attr == "href" end) do
           {_, href} ->
             text = Floki.text(children)
+
             if text == href do
               "[#{href}]"
             else
@@ -304,7 +307,7 @@ defmodule AWS.CodeGen.Docstring do
 
       {_, _attrs, children} ->
         "#{Floki.text(children)}"
-      end)
+    end)
     |> Floki.raw_html(encode: true)
   end
 

lib/aws_codegen/post_service.ex

@@ -1,11 +1,15 @@
 defmodule AWS.CodeGen.PostService do
   alias AWS.CodeGen.Docstring
   alias AWS.CodeGen.Service
+  alias AWS.CodeGen.Shapes
 
   defmodule Action do
     defstruct arity: nil,
               docstring: nil,
               function_name: nil,
+              input: nil,
+              output: nil,
+              errors: %{},
               host_prefix: nil,
               name: nil
   end
@@ -57,17 +61,24 @@ defmodule AWS.CodeGen.PostService do
     service = spec.api["shapes"][spec.shape_name]
     traits = service["traits"]
     actions = collect_actions(language, spec.api)
-    endpoint_prefix = traits["aws.api#service"]["endpointPrefix"] || traits["aws.api#service"]["arnNamespace"]
+    shapes = Shapes.collect_shapes(language, spec.api)
+
+    endpoint_prefix =
+      traits["aws.api#service"]["endpointPrefix"] || traits["aws.api#service"]["arnNamespace"]
+
     endpoint_info = endpoints_spec["services"][endpoint_prefix]
     is_global = not is_nil(endpoint_info) and not Map.get(endpoint_info, "isRegionalized", true)
+
     credential_scope =
       if is_global do
         endpoint_info["endpoints"]["aws-global"]["credentialScope"]["region"]
       end
+
     json_version = AWS.CodeGen.Util.get_json_version(service)
     protocol = spec.protocol |> to_string()
     content_type = @configuration[protocol][:content_type]
     content_type = content_type <> if protocol == "json", do: json_version, else: ""
+
     signing_name =
       if String.starts_with?(endpoint_prefix, "api.") do
         String.replace(endpoint_prefix, "api.", "")
@@ -89,6 +100,7 @@ defmodule AWS.CodeGen.PostService do
       language: language,
       module_name: spec.module_name,
       protocol: protocol |> to_string() |> String.replace("_", "-"),
+      shapes: shapes,
       signing_name: signing_name,
       signature_version: AWS.CodeGen.Util.get_signature_version(service),
       service_id: AWS.CodeGen.Util.get_service_id(service),
@@ -102,7 +114,9 @@ defmodule AWS.CodeGen.PostService do
     |> case do
       {key, _} ->
         String.replace(key, ~r/.*#/, "")
-      nil -> nil
+
+      nil ->
+        nil
     end
   end
 
@@ -113,12 +127,22 @@ defmodule AWS.CodeGen.PostService do
       Enum.reduce(shapes, [], fn {_, shape}, acc ->
         case shape["type"] do
           "service" ->
-
             [acc | List.wrap(shape["operations"])]
+
           "resource" ->
-            [shape["operations"], shape["collectionOperations"], shape["create"], shape["put"], shape["read"], shape["update"], shape["delete"], shape["list"]]
+            [
+              shape["operations"],
+              shape["collectionOperations"],
+              shape["create"],
+              shape["put"],
+              shape["read"],
+              shape["update"],
+              shape["delete"],
+              shape["list"]
+            ]
             |> Enum.reject(&is_nil/1)
             |> Kernel.++(acc)
+
           _ ->
             acc
         end
@@ -128,6 +152,7 @@ defmodule AWS.CodeGen.PostService do
 
     Enum.map(operations, fn operation ->
       operation_spec = shapes[operation]
+
       %Action{
         arity: 3,
         docstring:
@@ -137,7 +162,10 @@ defmodule AWS.CodeGen.PostService do
           ),
         function_name: AWS.CodeGen.Name.to_snake_case(operation),
         host_prefix: operation_spec["traits"]["smithy.api#endpoint"]["hostPrefix"],
-        name: String.replace(operation, ~r/com\.amazonaws\.[^#]+#/, "")
+        name: String.replace(operation, ~r/com\.amazonaws\.[^#]+#/, ""),
+        input: operation_spec["input"],
+        output: operation_spec["output"],
+        errors: operation_spec["errors"]
       }
     end)
     |> Enum.sort(fn a, b -> a.function_name < b.function_name end)

lib/aws_codegen/rest_service.ex

@@ -23,7 +23,10 @@ defmodule AWS.CodeGen.RestService do
               send_body_as_binary?: false,
               receive_body_as_binary?: false,
               host_prefix: nil,
-              language: nil
+              language: nil,
+              input: nil,
+              output: nil,
+              errors: []
 
     def method(action) do
       result = action.method |> String.downcase() |> String.to_atom()
@@ -117,7 +120,10 @@ defmodule AWS.CodeGen.RestService do
     traits = service["traits"]
     actions = collect_actions(language, spec.api)
     protocol = spec.protocol
-    endpoint_prefix = traits["aws.api#service"]["endpointPrefix"] || traits["aws.api#service"]["arnNamespace"] ##TODO: for some reason this field is not always present and docs are not clear on what to do
+    ## TODO: for some reason this field is not always present and docs are not clear on what to do
+    endpoint_prefix =
+      traits["aws.api#service"]["endpointPrefix"] || traits["aws.api#service"]["arnNamespace"]
+
     endpoint_info = endpoints_spec["services"][endpoint_prefix]
     is_global = not is_nil(endpoint_info) and not Map.get(endpoint_info, "isRegionalized", true)
 
@@ -145,7 +151,9 @@ defmodule AWS.CodeGen.RestService do
       signing_name: signing_name,
       signature_version: AWS.CodeGen.Util.get_signature_version(service),
       service_id: AWS.CodeGen.Util.get_service_id(service),
-      target_prefix: nil, ##TODO: metadata["targetPrefix"]
+      ## TODO: metadata["targetPrefix"],
+      target_prefix: nil,
+      shapes: Shapes.collect_shapes(language, spec.api)
     }
   end
 
@@ -163,6 +171,7 @@ defmodule AWS.CodeGen.RestService do
   """
   def function_parameters(action, required_only \\ false) do
     language = action.language
+
     Enum.join([
       join_parameters(action.url_parameters, language)
       | case action.method do
@@ -210,12 +219,22 @@ defmodule AWS.CodeGen.RestService do
       Enum.reduce(shapes, [], fn {_, shape}, acc ->
         case shape["type"] do
           "service" ->
-
             List.wrap(shape["operations"]) ++ acc
+
           "resource" ->
-            [shape["operations"], shape["collectionOperations"], shape["create"], shape["put"], shape["read"], shape["update"], shape["delete"], shape["list"]]
+            [
+              shape["operations"],
+              shape["collectionOperations"],
+              shape["create"],
+              shape["put"],
+              shape["read"],
+              shape["update"],
+              shape["delete"],
+              shape["list"]
+            ]
             |> Enum.reject(&is_nil/1)
             |> Kernel.++(acc)
+
           _ ->
             acc
         end
@@ -253,6 +272,7 @@ defmodule AWS.CodeGen.RestService do
 
       input_shape = Shapes.get_input_shape(operation_spec)
       output_shape = Shapes.get_output_shape(operation_spec)
+
       %Action{
         arity: length(url_parameters) + len_for_method,
         docstring:
@@ -275,7 +295,10 @@ defmodule AWS.CodeGen.RestService do
         send_body_as_binary?: Shapes.body_as_binary?(shapes, input_shape),
         receive_body_as_binary?: Shapes.body_as_binary?(shapes, output_shape),
         host_prefix: operation_spec["traits"]["smithy.api#endpoint"]["hostPrefix"],
-        language: language
+        language: language,
+        input: operation_spec["input"],
+        output: operation_spec["output"],
+        errors: operation_spec["errors"]
       }
     end)
     |> Enum.sort(fn a, b -> a.function_name < b.function_name end)
@@ -291,12 +314,16 @@ defmodule AWS.CodeGen.RestService do
   end
 
   defp collect_url_parameters(language, api_spec, operation) do
-    url_params = collect_parameters(language, api_spec, operation, "input", "smithy.api#httpLabel")
+    url_params =
+      collect_parameters(language, api_spec, operation, "input", "smithy.api#httpLabel")
+
     url_params
   end
 
   defp collect_query_parameters(language, api_spec, operation) do
-    query_params = collect_parameters(language, api_spec, operation, "input", "smithy.api#httpQueryParams")
+    query_params =
+      collect_parameters(language, api_spec, operation, "input", "smithy.api#httpQueryParams")
+
     params = collect_parameters(language, api_spec, operation, "input", "smithy.api#httpQuery")
     query_params ++ params
   end
@@ -311,13 +338,18 @@ defmodule AWS.CodeGen.RestService do
 
   defp collect_parameters(language, api_spec, operation, data_type, param_type) do
     shape_name = api_spec["shapes"][operation][data_type]["target"]
+
     if shape_name do
       case api_spec["shapes"][shape_name] do
         nil ->
           []
+
         shape ->
           required_members =
-            for {name, %{"traits" => traits}} <- shape["members"], Map.has_key?(traits, "smithy.api#required"), do: name
+            for {name, %{"traits" => traits}} <- shape["members"],
+                Map.has_key?(traits, "smithy.api#required"),
+                do: name
+
           shape["members"]
           |> Enum.filter(filter_fn(param_type))
           |> Enum.map(fn {name, x} ->
@@ -349,6 +381,7 @@ defmodule AWS.CodeGen.RestService do
       required: required
     }
   end
+
   defp build_parameter(language, {name, data}, required) do
     %Parameter{
       code_name:
@@ -362,5 +395,4 @@ defmodule AWS.CodeGen.RestService do
       required: required
     }
   end
-
 end

lib/aws_codegen/shapes.ex

@@ -1,5 +1,14 @@
 defmodule AWS.CodeGen.Shapes do
-  @moduledoc false
+  defmodule Shape do
+    defstruct name: nil,
+              type: nil,
+              members: [],
+              member: [],
+              enum: [],
+              min: nil,
+              required: [],
+              is_input: nil
+  end
 
   def get_input_shape(operation_spec) do
     get_in(operation_spec, ["input", "target"])
@@ -9,10 +18,27 @@ defmodule AWS.CodeGen.Shapes do
     get_in(operation_spec, ["output", "target"])
   end
 
+  def collect_shapes(_language, api_spec) do
+    api_spec["shapes"]
+    |> Map.new(fn {name, shape} ->
+      {name,
+       %Shape{
+         name: name,
+         type: shape["type"],
+         member: shape["member"],
+         members: shape["members"],
+         min: shape["min"],
+         enum: shape["enum"],
+         is_input: is_input?(shape)
+       }}
+    end)
+  end
+
   def body_as_binary?(shapes, shape) do
     ## TODO: Should we validate or search for trait `smithy.api#httpPayload` rather than
     ##  trust that the member is always named `Body`?
     inner_spec = get_in(shapes, [shape, "members", "Body"])
+
     if is_map(inner_spec) && Map.has_key?(inner_spec, "target") do
       ## TODO: we should extract the type from the actual shape `type` rather than infer it from the naming
       inner_spec["target"]
@@ -23,4 +49,7 @@ defmodule AWS.CodeGen.Shapes do
     end
   end
 
+  def is_input?(shape) do
+    !Map.has_key?(shape, "traits") or Map.has_key?(shape["traits"], "smithy.api#input")
+  end
 end