From 67b1de49f181847bf568ab0e262dcdf59416cdf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Bia=C5=82y?= Date: Wed, 10 Apr 2024 11:53:10 +0200 Subject: [PATCH] changes to docs for 0.3, first draft --- website/docs/architecture.md | 4 +- website/docs/async.md | 119 +++++++++++++++++++++++++++++ website/docs/basics.md | 11 ++- website/docs/constructors.md | 142 +++++++---------------------------- website/docs/io.md | 78 +++++++++++++++++++ website/sidebars.js | 2 + 6 files changed, 236 insertions(+), 120 deletions(-) create mode 100644 website/docs/async.md create mode 100644 website/docs/io.md diff --git a/website/docs/architecture.md b/website/docs/architecture.md index eda022d1..ab227cbf 100644 --- a/website/docs/architecture.md +++ b/website/docs/architecture.md @@ -2,7 +2,6 @@ title: Overview --- - Pulumi runtime is **asynchronous by design**. The goal is to allow the user's program to declare all the necessary resources as fast as possible so that Pulumi engine can make informed decisions about which parts of the deployment plan can be executed in parallel and therefore yield good performance. @@ -30,9 +29,10 @@ Besom stands alone in this choice and due to it **has some differences** in comp Following sections explain and showcase said differences: -- [Resource constructors](constructors.md) - resource constructors are pure functions that return Outputs - [Context](context.md) - context is passed around implicitly via Scala's Context Function - [Exports](exports.md) - your program is a function that returns Stack along with its Stack Outputs +- [Inputs and Outputs](io.md) - Outputs are static or dynamic properties passed to Inputs to configure resources +- [Resource constructors](constructors.md) - resource constructors are pure functions that return Outputs - [Laziness](laziness.md) - dangling resources are possible and resource constructors are memoized - [Apply method](apply_methods.md) - use `map` and `flatMap` to compose Outputs, not `apply` - [Logging](logging.md) - all logging statements need to be composed into the main flow diff --git a/website/docs/async.md b/website/docs/async.md new file mode 100644 index 00000000..1f80dc37 --- /dev/null +++ b/website/docs/async.md @@ -0,0 +1,119 @@ +--- +title: Resource constructor asynchronicity +--- +import Tabs from '@theme/Tabs'; +import TabItem from '@theme/TabItem'; + +Resources in Besom have an interesting property related to the fact that Pulumi's runtime is asynchronous. +One could suspect that in following snippet resources are created sequentially due to monadic syntax: + +```scala +for + a <- aws.s3.Bucket("first") + b <- aws.s3.Bucket("second") +yield () +``` + +This isn't true. Pulumi expects that a language SDK will declare resources as fast as possible. Due to this +fact resource constructors return immediately after they spawn a Resource object. A resource object is just a +plain case class with each property expressed in terms of Outputs. The work necessary for resolution of these +properties is executed asynchronously. In the example above both buckets will be created in parallel. + +Given that a piece of code is worth more than a 1000 words, below you can find code snippets that explain these +semantics using known Scala technologies. In each of them `Output` is replaced with a respective async datatype +to explain what internals of Besom are actually doing when resource constructors are called (oversimplified a +bit). + + + + +```scala +// internal function, here just to represent types +def resolveResourceAsync(name: String, args: Args, promises: Promise[_]*): Future[Unit] = ??? + +// resource definition +case class Bucket(bucket: Future[String]) +object Bucket: + def apply(name: String, args: BucketArgs = BucketArgs()): Future[Bucket] = + // create a promise for bucket property + val bucketNamePromise = Promise[String]() + // kicks off async resolution of the resource properties + resolveResourceAsync(name, args, bucketNamePromise) + // returns immediately + Future.successful(Bucket(bucketNamePromise.future)) + +// this just returns a Future[Unit] that will be resolved immediately +for + a <- Bucket("first") + b <- Bucket("second") +yield () +``` + + + + +```scala +// internal function, here just to represent types +def resolveResourceAsync(name: String, args: Args, promises: Deferred[IO, _]*): IO[Unit] = ??? + +// resource definition +case class Bucket(bucket: IO[String]) +object Bucket: + def apply(name: String, args: BucketArgs = BucketArgs()): IO[Bucket] = + for + // create a deferred for bucket property + bucketNameDeferred <- Deferred[IO, String]() + // kicks off async resolution of the resource properties + _ <- resolveResourceAsync(name, args, bucketNameDeferred).start + yield Bucket(bucketNameDeferred.get) // returns immediately + +// this just returns a IO[Unit] that will be resolved immediately +for + a <- Bucket("first") + b <- Bucket("second") +yield () +``` + + + + +```scala +// internal function, here just to represent types +def resolveResourceAsync(name: String, args: Args, promises: Promise[_]*): Task[Unit] = ??? + +// resource definition +case class Bucket(bucket: Task[String]) +object Bucket: + def apply(name: String, args: BucketArgs = BucketArgs()): Task[Bucket] = + for + // create a promise for bucket property + bucketNamePromise <- Promise.make[Exception, String]() + // kicks off async resolution of the resource properties + _ <- resolveResourceAsync(name, args, bucketNameDeferred).fork + yield Bucket(bucketNameDeferred.await) // returns immediately + +// this just returns a Task[Unit] that will be resolved immediately +for + a <- Bucket("first") + b <- Bucket("second") +yield () +``` + + + + +There is a way to inform Pulumi engine that some of the resources have to be created, updated or deleted +sequentially. To do that, one has to pass [resource options](https://www.pulumi.com/docs/concepts/options/) +to adequate resource constructors with `dependsOn` property set to resource to await for. Here's an example: +```scala +for + a <- Bucket("first") + b <- Bucket("second", BucketArgs(), opts(dependsOn = a)) +yield () +``` + + +:::info +A good observer will notice that all these forks have to be awaited somehow and that is true. Besom +does await for all spawned Outputs to be resolved before finishing the run. +::: \ No newline at end of file diff --git a/website/docs/basics.md b/website/docs/basics.md index c0a9590a..ee1a1e1d 100644 --- a/website/docs/basics.md +++ b/website/docs/basics.md @@ -218,7 +218,8 @@ Inputs and Outputs are the primary [asynchronous data types in Pulumi](https://www.pulumi.com/docs/concepts/inputs-outputs/), and they signify values that will be provided by the engine later, when the resource is created and its properties can be fetched. -`Input[A]` type is an alias for `Output[A]` type used by [resource](#resources) arguments. +`Input[A]` type is an alias for `Output[A]` type used by [resource](#resources) arguments. Inputs are +[very elastic](io.md/#inputs) in what they can receive to facilitate preview-friendly, declarative model of programming. Outputs are values of type `Output[A]` and behave very much like [monads](https://en.wikipedia.org/wiki/Monad_(functional_programming)). @@ -229,7 +230,7 @@ Outputs are used to: - automatically captures dependencies between [resources](#resources) - provide a way to express transformations on its value before it's known -- deffer the evaluation of its value until it's known +- defer the evaluation of its value until it's known - track the _secretness_ of its value Output transformations available in Besom: @@ -238,9 +239,11 @@ Output transformations available in Besom: output - [lifting](lifting.md) directly read properties off an output value - [interpolation](interpolator.md) concatenate string outputs with other strings directly -- `sequence` method combines multiple outputs into a single output of a list +- `sequence` method combines multiple outputs into a single output of a collection (`parSequence` variant is also available + for explicit parallel evaluation) - `zip` method combines multiple outputs into a single output of a tuple -- `traverse` method transforms a map of outputs into a single output of a map +- `traverse` method transforms a collection of values into a single output of a collection ((`parTraverse` variant is also available + for explicit parallel evaluation)) To create an output from a plain value, use the `Output` constructor, e.g.: diff --git a/website/docs/constructors.md b/website/docs/constructors.md index d3934882..3560e009 100644 --- a/website/docs/constructors.md +++ b/website/docs/constructors.md @@ -1,17 +1,11 @@ --- -title: Resource constructors and asynchronicity +title: Resource constructors --- -import Tabs from '@theme/Tabs'; -import TabItem from '@theme/TabItem'; ## Resources Resources are the [primary construct of Pulumi](basics.md#resources) programs. -## Outputs - -Outputs are the [primary asynchronous data structure of Pulumi](basics.md#inputs-and-outputs) programs. - ## Resource constructor syntax Most Pulumi SDKs expect you to create resource objects by invoking their constructors with `new` keyword, @@ -45,122 +39,42 @@ We have retained the CamelCase naming convention of resource constructors for pa You can always expect resource constructor names to start with capital letter. ::: -## Resource asynchronicity - -Resources in Besom have an interesting property related to the fact that Pulumi's runtime is asynchronous. -One could suspect that in following snippet resources are created sequentially due to monadic syntax: +Resource constructors always take 3 arguments: -```scala -for - a <- aws.s3.Bucket("first") - b <- aws.s3.Bucket("second") -yield () -``` + * resource name - this is an unique name of the resource in Pulumi's state management. Uniqueness is limited + to stack and the string has to be a `NonEmptyString` (most of the time Besom will be able to infer automatically + if `String` literal is non-empty but in case of ambiguity you can just add a type ascription `: NonEmptyString` + or, in case of dynamically obtained values, use `NonEmptyString`'s `apply` method that returns + `Option[NonEmptyString]`). -This isn't true. Pulumi expects that a language SDK will declare resources as fast as possible. Due to this -fact resource constructors return immediately after they spawn a Resource object. A resource object is just a -plain case class with each property expressed in terms of Outputs. The work necessary for resolution of these -properties is executed asynchronously. In the example above both buckets will be created in parallel. + * resource args - each resource has it's own companion args class (for instance, `aws.s3.Bucket` has a + `aws.s3.BucketArgs`) that takes `Input`s of values necessary to configure the resource. Args can be optional + when there are reasonable defaults and no input is necessary from the user. -Given that a piece of code is worth more than a 1000 words, below you can find code snippets that explain these -semantics using known Scala technologies. In each of them `Output` is replaced with a respective async datatype -to explain what internals of Besom are actually doing when resource constructors are called (oversimplified a -bit). + * resource options - [resource options](https://www.pulumi.com/docs/concepts/options/) are additional properties + that tell Pulumi engine how to apply changes related to this particular resource. Resource options are always + optional. These options dictate the order of creation (explicit dependency between resources otherwise unrelated + on data level), destruction, the way of performing updates (for instance delete and recreate) and which provider + instance to use. There are 3 types of resource options: + * `CustomResourceOptions` used with most of the resources defined in provider packages + * `ComponentResourceOptions` used with resources, both user-defined and remote components (defined in provider packages) + and finally + * `StackReferenceResourceOptions` used with [StackReferences](basics.md/#stack-references). + + To ease the use of resource options a shortcut context function is provided that allows user to summon the constructor + of expected resource options type by just typing `opts(...)`. Here's an example: - - ```scala -// internal function, here just to represent types -def resolveResourceAsync(name: String, args: Args, promises: Promise[_]*): Future[Unit] = ??? - -// resource definition -case class Bucket(bucket: Future[String]) -object Bucket: - def apply(name: String, args: BucketArgs = BucketArgs()): Future[Bucket] = - // create a promise for bucket property - val bucketNamePromise = Promise[String]() - // kicks off async resolution of the resource properties - resolveResourceAsync(name, args, bucketNamePromise) - // returns immediately - Future.successful(Bucket(bucketNamePromise.future)) - -// this just returns a Future[Unit] that will be resolved immediately -for - a <- Bucket("first") - b <- Bucket("second") -yield () -``` - - - +val myAwsProvider: Output[aws.Provider] = aws.Provider("my-aws-provider", aws.ProviderArgs(...)) -```scala -// internal function, here just to represent types -def resolveResourceAsync(name: String, args: Args, promises: Deferred[IO, _]*): IO[Unit] = ??? - -// resource definition -case class Bucket(bucket: IO[String]) -object Bucket: - def apply(name: String, args: BucketArgs = BucketArgs()): IO[Bucket] = - for - // create a deferred for bucket property - bucketNameDeferred <- Deferred[IO, String]() - // kicks off async resolution of the resource properties - _ <- resolveResourceAsync(name, args, bucketNameDeferred).start - yield Bucket(bucketNameDeferred.get) // returns immediately - -// this just returns a IO[Unit] that will be resolved immediately -for - a <- Bucket("first") - b <- Bucket("second") -yield () +val s3Bucket: Output[aws.s3.Bucket] = aws.s3.Bucket( + "my-bucket", + aws.s3.BucketArgs(...), + opts(provider = myAwsProvider) +) ``` - - - -```scala -// internal function, here just to represent types -def resolveResourceAsync(name: String, args: Args, promises: Promise[_]*): Task[Unit] = ??? - -// resource definition -case class Bucket(bucket: Task[String]) -object Bucket: - def apply(name: String, args: BucketArgs = BucketArgs()): Task[Bucket] = - for - // create a promise for bucket property - bucketNamePromise <- Promise.make[Exception, String]() - // kicks off async resolution of the resource properties - _ <- resolveResourceAsync(name, args, bucketNameDeferred).fork - yield Bucket(bucketNameDeferred.await) // returns immediately - -// this just returns a Task[Unit] that will be resolved immediately -for - a <- Bucket("first") - b <- Bucket("second") -yield () -``` - - - - -There is a way to inform Pulumi engine that some of the resources have to be created, updated or deleted -sequentially. To do that, one has to pass [resource options](https://www.pulumi.com/docs/concepts/options/) -to adequate resource constructors with `dependsOn` property set to resource to await for. Here's an example: -```scala -for - a <- Bucket("first") - b <- Bucket("second", BucketArgs(), opts(dependsOn = a)) -yield () -``` - - -:::info -A good observer will notice that all these forks have to be awaited somehow and that is true. Besom -does await for all spawned Outputs to be resolved before finishing the run. -::: - ## Compile time checking Besom tries to catch as many errors as possible at compile time, examples of our compile time checks are: diff --git a/website/docs/io.md b/website/docs/io.md new file mode 100644 index 00000000..caf815cb --- /dev/null +++ b/website/docs/io.md @@ -0,0 +1,78 @@ +--- +title: Inputs and Outputs +--- + +Outputs are the [primary asynchronous data structure of Pulumi](basics.md#inputs-and-outputs) programs. + +### Outputs + +Outputs are: + * pure and lazy - meaning that they suspend evaluation of code until evaluation, which is perfomed by Besom runtime + that runs `Pulumi.run` function at the, so called, end-of-the-world. + + * monadic - meaning that they expose `map` and `flatMap` operators and can be used in for-comprehensions + +Outputs are capable of consuming other effects for which there exists an instance of `ToFuture` typeclass. Besom +provides 3 such instances: + +- package `besom-core` provides an instance for `scala.concurrent.Future` +- package `besom-cats` provides an instance for `cats.effect.IO` +- package `besom-zio` provides an instance for `zio.Task` + +### Inputs + +Inputs are Besom types used wherever a value is expected to be provided by user primarily to ease the use of the +configuration necessary for resource constructors to spawn infrastructure resources. Inputs allow user to provide both +raw values, values that are wrapped in an `Output`, both of the former or nothing at all when dealing with optional +fields or even singular raw values or lists of values for fields that expect multiple values. + +To make this more digestable - the basic `Input[A]` type is declared as: + +```scala +opaque type Input[+A] >: A | Output[A] = A | Output[A] +``` + +ane the `Input.Optional[A]` variant is declared as: + +```scala +opaque type Optional[+A] >: Input[A | Option[A]] = Input[A | Option[A]] +``` + +This allows for things like this: + +```scala +val int1: Input[Int] = 23 +val int2: Input[Int] = Output(23) +// what if it's an optional value? +val maybeInt1: Input.Optional[Int] = 23 +val maybeInt2: Input.Optional[Int] = None +val maybeInt3: Input.Optional[Int] = Some(23) +// yes, but also: +val maybeInt4: Input.Optional[Int] = Output(23) +val maybeInt5: Input.Optional[Int] = Output(Option(23)) +val maybeInt6: Input.Optional[Int] = Output(None) +``` + +This elastic and permissive model was designed to allow a more declarative style and facilitate the implicit +parallelism of evaluation. In fact, Outputs are meant to be thought of as short pipelines that one uses +to transform properties and values obtained from one resource to be used as argument for another. If you're +used to the classic way of working with monadic programs with chains of `flatMap` and `map` or for-comprehensions +this might seem a bit odd to you - why would we take values wrapped in Outputs as arguments? The answer is: previews! + +Outputs incorporate semantics of `Option` to support Pulumi's preview / dry-run feature that allows one to see what +changes will be applied when the program is executed against the actual environment. This, however, means that Outputs +can short-circuit when a computed (provided by the engine) value is missing in dry-run and most properties on resources +belong to this category. It is entirely possible to structure a Besom program the same way one would structure a program +that uses Cats Effect IO or ZIO but once you `flatMap` on an Output value that can be only obtained from actual environment +short-circuiting logic will kick in and all the subsequent `flatMap`/`map` steps will be skipped yielding a broken view +of the changes that will get applied in your next change to the infrastructure. To avoid this problem it is highly +recommended to write Besom programs in a style highly reminiscent of direct style and use for-comprehensions only to transform +properties passed from configuration or declared resources to another resources. This way the graph of resources is fully +known in dry-run phase and can be properly inspected. Full power of monadic composition should be reserved for situations +where it is strictly necessary. + +:::info +We are working on a solution that would allow us to track computed `Output` values on the type level and therefore inform +the user (via a compile-time information or warning) that a dynamic subtree of resources will be spawned by their code +that won't be visible in preview. +::: \ No newline at end of file diff --git a/website/sidebars.js b/website/sidebars.js index 788024e2..0c0906eb 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -43,7 +43,9 @@ const sidebars = { 'architecture', 'context', 'exports', + 'io', 'constructors', + 'async', 'laziness', 'apply_methods', 'logging',