diff --git a/.convco/template/commit.hbs b/.convco/template/commit.hbs new file mode 100644 index 00000000..6e080e80 --- /dev/null +++ b/.convco/template/commit.hbs @@ -0,0 +1,14 @@ +* +{{~#if scope}} **{{scope}}:** +{{~/if}} +{{#if references}} + {{~#each references}} {{#if @root.linkReferences~}} + [{{this.prefix}}{{this.issue}}]({{issueUrlFormat}}) + {{~else}}{{this.prefix}}{{this.issue}} + {{~/if}}{{/each}} +{{~/if}} + {{subject}} +{{~#if hash}} {{#if @root.linkReferences}}([{{shortHash}}]({{commitUrlFormat}})){{else}}({{hash}}) +{{~/if}} +{{~/if}} + diff --git a/.convco/template/footer.hbs b/.convco/template/footer.hbs new file mode 100644 index 00000000..e69de29b diff --git a/.convco/template/header.hbs b/.convco/template/header.hbs new file mode 100644 index 00000000..a4659a61 --- /dev/null +++ b/.convco/template/header.hbs @@ -0,0 +1,2 @@ + +{{#if isPatch}}###{{else}}##{{/if}}{{#if @root.linkCompare}} [{{version}}]({{compareUrlFormat}}){{else}} {{version}}{{/if}}{{#if title}} "{{title}}"{{/if}}{{#if date}} ({{date}}){{/if}} \ No newline at end of file diff --git a/.convco/template/template.hbs b/.convco/template/template.hbs new file mode 100644 index 00000000..eaae1dc2 --- /dev/null +++ b/.convco/template/template.hbs @@ -0,0 +1,19 @@ +{{> header}} +{{#if noteGroups}}{{#each noteGroups}} + +## ⚠ {{title}} + +{{#each notes}}* {{#if commit.scope}}**{{commit.scope}}:** {{/if}}{{this.text}} +{{/each}} +{{/each}} +{{/if}} + +{{#each commitGroups}} +{{#if title}}{{#if @root.isPatch}} +##{{else}} +##{{/if}} {{title}}{{/if}} + +{{#each commits}} +{{> commit root=@root}} +{{/each}} +{{/each}} \ No newline at end of file diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md deleted file mode 100644 index 5d141053..00000000 --- a/.github/pull_request_template.md +++ /dev/null @@ -1,10 +0,0 @@ - -**Checklist** -- [ ] Updated CHANGELOG.md describing pertinent changes. -- [ ] Updated README.md with pertinent info (may not always apply). -- [ ] Updated `site` content with pertinent info (may not always apply). -- [ ] Squash down commits to one or two logical commits which clearly describe the work you've done. If you don't, then Dodd will 🤓. diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 18fa5cf2..3220d1ff 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1,14 +1,20 @@ name: CI -on: [push, pull_request] +on: + push: + branches: + - master + pull_request: + branches: + - master jobs: lint: name: lint runs-on: ubuntu-latest steps: - name: Setup | Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - - uses: actions/cache@v2 + - uses: actions/cache@v3 with: path: | ~/.cargo/registry @@ -19,24 +25,13 @@ jobs: ${{ runner.os }}-lint - name: Setup | Toolchain (clippy) - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - default: true - components: clippy + run: | + rustup toolchain install stable --component clippy + rustup default stable - name: Build | Clippy - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --all-targets - - - name: Setup | Toolchain (rustfmt) - uses: actions-rs/toolchain@v1 - with: - toolchain: nightly - default: true - components: rustfmt + run: | + cargo clippy --all-targets --all-features -- -D warnings - name: Build | Rustfmt run: cargo fmt -- --check @@ -45,10 +40,10 @@ jobs: runs-on: ubuntu-latest steps: - name: Setup | Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup | Cache Cargo - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cargo/registry @@ -59,10 +54,9 @@ jobs: ${{ runner.os }}-check - name: Setup | Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - profile: minimal + run: | + rustup toolchain install stable + rustup default stable - name: Build | Check run: cargo check --all @@ -83,10 +77,10 @@ jobs: runs-on: ${{ matrix.os }} steps: - name: Setup | Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup | Cache Cargo - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cargo/registry @@ -97,7 +91,7 @@ jobs: ${{ runner.os }}-cargo - name: Setup | Cache | Example proxy - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: examples/proxy/target key: wasm32-example-proxy @@ -105,7 +99,7 @@ jobs: wasm32-example-proxy - name: Setup | Cache | Example seed - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: examples/seed/target key: wasm32-example-seed @@ -113,7 +107,7 @@ jobs: wasm32-example-seed - name: Setup | Cache | Example vanilla - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: examples/vanilla/target key: wasm32-example-vanilla @@ -121,7 +115,7 @@ jobs: wasm32-example-vanilla - name: Setup | Cache | Example webworker - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: examples/webworker/target key: wasm32-example-webworker @@ -129,7 +123,7 @@ jobs: wasm32-example-webworker - name: Setup | Cache | Example webworker-gloo - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: examples/webworker-gloo/target key: wasm32-example-webworker-gloo @@ -137,7 +131,7 @@ jobs: wasm32-example-webworker-gloo - name: Setup | Cache | Example yew - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: examples/yew/target key: wasm32-example-yew @@ -145,19 +139,25 @@ jobs: wasm32-example-yew - name: Setup | Cache | Example yew-tailwindcss - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: examples/yew-tailwindcss/target key: wasm32-example-yew-tailwindcss restore-keys: | wasm32-example-yew-tailwindcss - - name: Setup | Rust - uses: actions-rs/toolchain@v1 + - name: Setup | Cache | Example yew-tls + uses: actions/cache@v3 with: - toolchain: stable - profile: minimal - target: wasm32-unknown-unknown + path: examples/yew-tls/target + key: wasm32-example-yew-tls + restore-keys: | + wasm32-example-yew-tls + + - name: Setup | Rust + run: | + rustup toolchain install stable --target wasm32-unknown-unknown + rustup default stable - name: Build | Test run: cargo test @@ -181,5 +181,7 @@ jobs: run: ${{ matrix.binPath }} --config=examples/yew/Trunk.toml build - name: Build | Examples | yew-tailwindcss run: ${{ matrix.binPath }} --config=examples/yew-tailwindcss/Trunk.toml build + - name: Build | Examples | yew-tls + run: ${{ matrix.binPath }} --config=examples/yew-tls/Trunk.toml build - name: Build | Examples | no-rust run: ${{ matrix.binPath }} --config=examples/no-rust/Trunk.toml build diff --git a/.github/workflows/pages.yaml b/.github/workflows/pages.yaml index c0da8f1f..db928693 100644 --- a/.github/workflows/pages.yaml +++ b/.github/workflows/pages.yaml @@ -1,15 +1,21 @@ name: Pages on: - push: - tags: - - "v*" + # Only run after a successful release + workflow_call: + # Allow to run manually + workflow_dispatch: + +permissions: + contents: read + pages: write + id-token: write jobs: deploy: runs-on: ubuntu-latest steps: - name: Setup | Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 with: submodules: recursive - name: Setup | Zola @@ -23,4 +29,4 @@ jobs: with: github_token: ${{ secrets.GITHUB_TOKEN }} publish_dir: ./site/public - cname: trunkrs.dev + cname: trunkrs.dev \ No newline at end of file diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index d729ab93..4acbce1f 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -5,31 +5,70 @@ on: - "v*" jobs: + init: + runs-on: ubuntu-22.04 + outputs: + version: ${{steps.version.outputs.version}} + prerelease: ${{steps.state.outputs.prerelease}} + steps: + - name: Evaluate state + id: state + env: + HEAD_REF: ${{github.head_ref}} + run: | + test -z "${HEAD_REF}" && (echo 'do-publish=true' >> $GITHUB_OUTPUT) + if [[ "${{ github.event.ref }}" =~ ^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo release=true >> $GITHUB_OUTPUT + elif [[ "${{ github.event.ref }}" =~ ^refs/tags/v.*$ ]]; then + echo prerelease=true >> $GITHUB_OUTPUT + fi + - name: Set version + id: version + run: | + VERSION=$(echo "${{ github.ref }}" | sed -e 's,.*/\(.*\),\1,') + [[ "${{ github.ref }}" == "refs/tags/"* ]] && VERSION=$(echo $VERSION | sed -e 's/^v//') + [ "$VERSION" == "main" ] && VERSION=latest + echo "Version: $VERSION" + echo "version=$VERSION" >> $GITHUB_OUTPUT + build: strategy: fail-fast: false matrix: target: - x86_64-unknown-linux-gnu + - aarch64-unknown-linux-gnu - x86_64-apple-darwin - x86_64-pc-windows-msvc + - aarch64-apple-darwin include: - target: x86_64-unknown-linux-gnu - os: ubuntu-latest + os: ubuntu-22.04 name: trunk-x86_64-unknown-linux-gnu.tar.gz + - target: aarch64-unknown-linux-gnu + os: ubuntu-22.04 + name: trunk-aarch64-unknown-linux-gnu.tar.gz + cross: "true" - target: x86_64-apple-darwin - os: macos-latest + os: macos-12 name: trunk-x86_64-apple-darwin.tar.gz + - target: aarch64-apple-darwin + os: macos-12 + xcode: "true" + name: trunk-aarch64-apple-darwin.tar.gz - target: x86_64-pc-windows-msvc - os: windows-latest + os: windows-2022 name: trunk-x86_64-pc-windows-msvc.zip + ext: ".exe" + runs-on: ${{ matrix.os }} + steps: - name: Setup | Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v4 - name: Setup | Cache Cargo - uses: actions/cache@v2 + uses: actions/cache@v3 with: path: | ~/.cargo/registry @@ -37,79 +76,143 @@ jobs: key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} - name: Setup | Rust - uses: actions-rs/toolchain@v1 - with: - toolchain: stable - override: true - profile: minimal - target: ${{ matrix.target }} + run: | + rustup toolchain install stable --target ${{ matrix.target }} --profile minimal + rustup default stable + + - name: Setup | Cross + if: matrix.cross == 'true' + run: | + curl -sSL https://github.com/cargo-bins/cargo-binstall/releases/latest/download/cargo-binstall-x86_64-unknown-linux-musl.tgz -o binstall.tar.gz + tar xzf binstall.tar.gz + mv cargo-binstall $HOME/.cargo/bin/ + cargo binstall cross -y - name: Build | Build - run: cargo build --release --target ${{ matrix.target }} + shell: bash + run: | + if [[ "${{ matrix.xcode }}" == "true" ]]; then + export SDKROOT=$(xcrun -sdk macosx --show-sdk-path) + export MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version) + fi + + CMD="cargo" + + if [[ -n "${{ matrix.cross }}" ]]; then + CMD="cross" + fi + + OPTS="--release" + + if [[ -n "${{ matrix.target }}" ]]; then + OPTS="$OPTS --target=${{ matrix.target }}" + fi + + ${CMD} build ${OPTS} - - name: Post Setup | Prepare artifacts [Windows] + - name: Post Build | List output + shell: bash + run: | + ls -l target/ + + - name: Post Build | Move binary + shell: bash + run: | + mkdir -p upload + + # if we have an alternate target, there is a sub-directory + if [[ -f "target/release/trunk${{ matrix.ext }}" ]]; then + SRC="target/release/trunk${{ matrix.ext }}" + elif [[ -f "target/${{ matrix.target }}/release/trunk${{ matrix.ext }}" ]]; then + SRC="target/${{ matrix.target }}/release/trunk${{ matrix.ext }}" + else + echo "Unable to find output" + find target + false # stop build + fi + + # for upload + cp -pv "${SRC}" upload/trunk${{ matrix.ext }} + + - name: Post Build | Strip binary + if: matrix.cross != 'true' + working-directory: upload + run: | + ls -l trunk${{matrix.ext}} + strip trunk${{matrix.ext}} + ls -l trunk${{matrix.ext}} + + - name: Post Build | Prepare artifacts [Windows] if: matrix.os == 'windows-latest' + working-directory: upload run: | - cd target/${{ matrix.target }}/release - strip trunk.exe - 7z a ../../../${{ matrix.name }} trunk.exe - cd - - - name: Post Setup | Prepare artifacts [-nix] + 7z a ${{ matrix.name }} trunk${{matrix.ext}} + + - name: Post Build | Prepare artifacts [-nix] if: matrix.os != 'windows-latest' + working-directory: upload run: | - cd target/${{ matrix.target }}/release - strip trunk - tar czvf ../../../${{ matrix.name }} trunk - cd - - - name: Post Setup | Upload artifacts - uses: actions/upload-artifact@v2 - with: - name: ${{ matrix.name }} - path: ${{ matrix.name }} + tar czvf ${{ matrix.name }} trunk${{matrix.ext}} - publish: - needs: build - runs-on: ubuntu-latest - steps: - - name: Setup | Checkout - uses: actions/checkout@v2 - - - name: Setup | Rust - uses: actions-rs/toolchain@v1 + - name: Post Build | Upload artifacts + uses: actions/upload-artifact@v3 with: - toolchain: stable - profile: minimal - override: true - - - name: Build | Publish - run: cargo publish --token ${{ secrets.CRATES_IO_TOKEN }} + name: ${{ matrix.name }} + path: upload/${{ matrix.name }} + if-no-files-found: error release: - needs: publish + needs: [init, build] runs-on: ubuntu-latest steps: - name: Setup | Checkout - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 0 - name: Setup | Artifacts - uses: actions/download-artifact@v2 + uses: actions/download-artifact@v3 - name: Setup | Checksums run: for file in trunk-*/trunk-*; do openssl dgst -sha256 -r "$file" | awk '{print $1}' > "${file}.sha256"; done - - name: Setup | Create Release Log - run: cat CHANGELOG.md | tail -n +7 | head -n 25 > RELEASE_LOG.md + - name: Setup | Install convco + run: | + curl -sLO https://github.com/convco/convco/releases/download/v0.4.2/convco-ubuntu.zip + unzip convco-ubuntu.zip + chmod a+x convco + sudo mv convco /usr/local/bin - - name: Build | Publish Pre-Release - uses: softprops/action-gh-release@v1 - with: - files: trunk-*/trunk-* - body_path: RELEASE_LOG.md - draft: true + - name: Setup | Generate changelog + run: | + convco changelog -s --max-majors=1 --max-minors=1 --max-patches=1 > RELEASE_LOG.md + + - name: Build | Publish env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + TAG: v${{ needs.init.outputs.version }} + run: | + OPTS="" + if [[ "${{ needs.init.outputs.prerelease }}" == "true" ]]; then + OPTS="${OPTS} -p" + fi + gh release create ${OPTS} --title "${{ needs.init.outputs.version }}" -F RELEASE_LOG.md ${TAG} \ + trunk-*/trunk-* + + pages: + needs: release + uses: ./.github/workflows/pages.yaml + + publish: + needs: pages + runs-on: ubuntu-latest + steps: + - name: Setup | Checkout + uses: actions/checkout@v4 + + - name: Build | Publish + run: cargo publish --token ${{ secrets.CRATES_IO_TOKEN }} + brew: needs: release diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml deleted file mode 100644 index 57aa8248..00000000 --- a/.github/workflows/stale.yaml +++ /dev/null @@ -1,20 +0,0 @@ -name: Stale -on: - schedule: - # Daily staleness check. - - cron: '0 0 * * *' - -jobs: - stale: - runs-on: ubuntu-latest - steps: - - uses: actions/stale@v8 - with: - stale-issue-message: 'This issue is stale because it has been open 30 days with no activity. Remove stale label or comment or this will be closed in 5 days.' - stale-pr-message: 'This PR is stale because it has been open 45 days with no activity. Remove stale label or comment or this will be closed in 10 days.' - close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity.' - close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity.' - days-before-issue-stale: 30 - days-before-pr-stale: 45 - days-before-issue-close: 5 - days-before-pr-close: 10 diff --git a/.gitignore b/.gitignore index 5329fcf5..9247a8bf 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ target dist site/public/* .vscode -.idea/ \ No newline at end of file +.idea/ +.DS_Store \ No newline at end of file diff --git a/.versionrc b/.versionrc new file mode 100644 index 00000000..579958bc --- /dev/null +++ b/.versionrc @@ -0,0 +1,4 @@ +template: ".convco/template" +linkReferences: false +linkCompare: true +header: "" \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md deleted file mode 100644 index 5d9d9507..00000000 --- a/CHANGELOG.md +++ /dev/null @@ -1,295 +0,0 @@ -changelog -========= -This changelog follows the patterns described here: https://keepachangelog.com/en/1.0.0/. - -Subheadings to categorize changes are `added, changed, deprecated, removed, fixed, security`. - -## Unreleased -### added -- Add options `accept_invalid_certs` and `root_certificate` to allow functioning behind corporate vpn connections -### changed -- Remove HTML glob in tailwind.config.js - -## 0.17.4 -### added -- Multiple PRs have been landed in attempts to address the recursive / infinite build cycle which can be triggered under some conditions. Shoutout to @ctron for their work in driving this resolution forward. - -## 0.17.3 -### added -- Add `inject_scripts` option to build configuration to allow toggle of injecting the modulepreload and scripts rendered in the final html. - -## 0.17.2 -### fixed -- Add missing `tools.tailwindcss` setting to configure the version of the Tailwindcss CLI to download. - -### added -- A few site updates. -- Some additional tests. - -## 0.17.1 -### changed -- Updated the default tool versions for wasm-bindgen, tailwind, sass, and wasm-opt. -- Update Trunk deps (should not have any functional implications). -- Update deps of all example projects. -- When resolving tools, do not abort if entries in the tool's extra paths list do not exist. This will happen naturally as part of the development of tools. Instead, we just log a warning. - -## 0.17.0 -### added -- Added `data-target-path` to `copy-dir`. -- Allow processing ` @@ -66,20 +102,29 @@ Trunk uses a source HTML file to drive all asset building and bundling. Trunk al The contents of your `dist` dir are now ready to be served on the web. +## JavaScript interoperability + +Trunk will create the necessary JavaScript code to bootstrap and run the WebAssembly based application. It will also +include all JavaScript snippets generated by `wasm-bindgen` for interfacing with JavaScript functionality. + +By default, functions exported from Rust, using `wasm-bingen`, can be accessed in the JavaScript code through the global +variable `window.wasmBindings`. This behavior can be disabled, and the name can be customized. + # Next Steps + That's not all! Trunk has even more useful features. Head on over to the following sections to learn more about how to use Trunk effectively. - [Assets](@/assets.md): learn about all of Trunk's supported asset types. - [Configuration](@/configuration.md): learn about Trunk's configuration system and how to use the Trunk proxy. - [Commands](@/commands.md): learn about Trunk's CLI commands for use in your development workflows. -- Join us on Discord by following this link Discord Chat +- Join us on Discord by following this link [![](https://img.shields.io/discord/793890238267260958?logo=discord&style=flat-square "Discord Chat")](https://discord.gg/JEPdBujTDr) # Contributing -Anyone and everyone is welcome to contribute! Please review the [CONTRIBUTING.md](https://github.com/thedodd/trunk/blob/master/CONTRIBUTING.md) document for more details. The best way to get started is to find an open issue, and then start hacking on implementing it. Letting other folks know that you are working on it, and sharing progress is a great approach. Open pull requests early and often, and please use GitHub's draft pull request feature. + +Anyone and everyone is welcome to contribute! Please review the [CONTRIBUTING.md](https://github.com/trunk-rs/trunk/blob/master/CONTRIBUTING.md) document for more details. The best way to get started is to find an open issue, and then start hacking on implementing it. Letting other folks know that you are working on it, and sharing progress is a great approach. Open pull requests early and often, and please use GitHub's draft pull request feature. # License -

- -
- trunk is licensed under the terms of the MIT License or the Apache License 2.0, at your choosing. -

+ +![](https://img.shields.io/badge/license-MIT%2FApache--2.0-blue?style=flat-square "license badge") + +trunk (as well as trunk) is licensed under the terms of the MIT License or the Apache License 2.0, at your choosing. diff --git a/site/content/assets.md b/site/content/assets.md index 051c6f42..6d4ab0b5 100644 --- a/site/content/assets.md +++ b/site/content/assets.md @@ -6,18 +6,20 @@ weight = 1 Declaring assets to be processed by Trunk is simple and extensible. -Trunk is still a young project, and new asset types will be added as we move forward. Keep an eye on [trunk#3](https://github.com/thedodd/trunk/issues/3) for more information on planned asset types, implementation status, and please contribute to the discussion if you think something is missing. - # Link Asset Types + All link assets to be processed by Trunk must follow these three rules: -- Must be declared as a valid HTML `link` tag. -- Must have the attribute `data-trunk`. -- Must have the attribute `rel="{type}"`, where `{type}` is one of the asset types listed below. + + - Must be declared as a valid HTML `link` tag. + - Must have the attribute `data-trunk`. + - Must have the attribute `rel="{type}"`, where `{type}` is one of the asset types listed below. This will typically look like: ``. Each asset type described below specifies the required and optional attributes for its asset type. All `` HTML elements will be replaced with the output HTML of the associated pipeline. ## rust + ✅ `rel="rust"`: Trunk will compile the specified Cargo project as WASM and load it. This is optional. If not specified, Trunk will look for a `Cargo.toml` in the parent directory of the source HTML file. + - `href`: (optional) the path to the `Cargo.toml` of the Rust project. If a directory is specified, then Trunk will look for the `Cargo.toml` in the given directory. If no value is specified, then Trunk will look for a `Cargo.toml` in the parent directory of the source HTML file. - `data-bin`: (optional) the name of the binary to compile and load. If the Cargo project has multiple binaries, this value will be required for proper functionality. - `data-type`: (optional) specifies how the binary should be loaded into the project. Can be set to `main` or `worker`. `main` is the default. There can only be one `main` link. For workers a wasm-bindgen javascript wrapper and the wasm file (with `_bg.wasm` suffix) is created, named after the binary name (if provided) or project name. See one of the webworker examples on how to load them. @@ -32,58 +34,100 @@ This will typically look like: ``). ## sass/scss + ✅ `rel="sass"` or `rel="scss"`: Trunk uses the official [dart-sass](https://github.com/sass/dart-sass) for compilation. Just link to your sass files from your source HTML, and Trunk will handle the rest. This content is hashed for cache control. The `href` attribute must be included in the link pointing to the sass/scss file to be processed. -- `data-inline`: (optional) this attribute will inline the compiled CSS from the SASS/SCSS file into a `"#, self.content), + ContentType::Css => format!(r#""#, self.content), ContentType::Js => format!(r#""#, self.content), + ContentType::Module => format!(r#""#, self.content), }; dom.select(&super::trunk_id_selector(self.id)) diff --git a/src/pipelines/js.rs b/src/pipelines/js.rs index 39fc43f6..9a714f0f 100644 --- a/src/pipelines/js.rs +++ b/src/pipelines/js.rs @@ -1,15 +1,18 @@ //! JS asset pipeline. -use std::path::PathBuf; -use std::sync::Arc; - +use super::{AssetFile, AttrWriter, Attrs, TrunkAssetPipelineOutput, ATTR_INTEGRITY, ATTR_SRC}; +use crate::{ + config::RtcBuild, + pipelines::AssetFileType, + processing::integrity::{IntegrityType, OutputDigest}, +}; use anyhow::{Context, Result}; use nipper::Document; +use std::path::PathBuf; +use std::str::FromStr; +use std::sync::Arc; use tokio::task::JoinHandle; -use super::{AssetFile, Attrs, TrunkAssetPipelineOutput, ATTR_SRC}; -use crate::config::RtcBuild; - /// A JS asset pipeline. pub struct Js { /// The ID of this pipeline's source HTML element. @@ -20,6 +23,8 @@ pub struct Js { asset: AssetFile, /// The attributes to be placed on the output script tag. attrs: Attrs, + /// The required integrity setting + integrity: IntegrityType, } impl Js { @@ -36,16 +41,25 @@ impl Js { let mut path = PathBuf::new(); path.extend(src_attr.split('/')); let asset = AssetFile::new(&html_dir, path).await?; + + let integrity = attrs + .get(ATTR_INTEGRITY) + .map(|value| IntegrityType::from_str(value)) + .transpose()? + .unwrap_or_default(); + // Remove src and data-trunk from attributes. let attrs = attrs .into_iter() .filter(|(x, _)| *x != "src" && !x.starts_with("data-trunk")) .collect(); + Ok(Self { id, cfg, asset, attrs, + integrity, }) } @@ -62,26 +76,32 @@ impl Js { tracing::info!(path = ?rel_path, "copying & hashing js"); let file = self .asset - .copy(&self.cfg.staging_dist, self.cfg.filehash) + .copy( + &self.cfg.staging_dist, + self.cfg.filehash, + self.cfg.release, + AssetFileType::Js, + ) .await?; tracing::info!(path = ?rel_path, "finished copying & hashing js"); - let attrs = Self::attrs_to_string(self.attrs); + + let result_file = self.cfg.staging_dist.join(&file); + let integrity = OutputDigest::generate(self.integrity, || std::fs::read(&result_file)) + .with_context(|| { + format!( + "Failed to generate digest for CSS file '{}'", + result_file.display() + ) + })?; + Ok(TrunkAssetPipelineOutput::Js(JsOutput { cfg: self.cfg.clone(), id: self.id, file, - attrs, + attrs: self.attrs, + integrity, })) } - - /// Convert attributes to a string, to be used in JsOutput. - fn attrs_to_string(attrs: Attrs) -> String { - attrs - .into_iter() - .map(|(k, v)| format!("{k}=\"{v}\"")) - .collect::>() - .join(" ") - } } /// The output of a JS build pipeline. @@ -93,15 +113,20 @@ pub struct JsOutput { /// Name of the finalized output file. pub file: String, /// The attributes to be added to the script tag. - pub attrs: String, + pub attrs: Attrs, + /// The digest for the integrity attribute + pub integrity: OutputDigest, } impl JsOutput { pub async fn finalize(self, dom: &mut Document) -> Result<()> { + let mut attrs = self.attrs.clone(); + self.integrity.insert_into(&mut attrs); + dom.select(&super::trunk_script_id_selector(self.id)) .replace_with_html(format!( - r#""#, - base = base, - js = js, - wasm = wasm, - ) - } - }; - match self.id { - Some(id) => dom - .select(&super::trunk_id_selector(id)) - .replace_with_html(script), - None => dom.select(body).append_html(script), - } - Ok(()) - } -} - /// Different optimization levels that can be configured with `wasm-opt`. #[derive(PartialEq, Eq)] enum WasmOptLevel { @@ -769,3 +811,10 @@ fn check_target_not_found_err(err: anyhow::Error, target: &str) -> anyhow::Error _ => err, } } + +/// Integrity of outputs +#[derive(Debug, Default)] +pub struct IntegrityOutput { + pub wasm: OutputDigest, + pub js: OutputDigest, +} diff --git a/src/pipelines/rust/output.rs b/src/pipelines/rust/output.rs new file mode 100644 index 00000000..d1807d82 --- /dev/null +++ b/src/pipelines/rust/output.rs @@ -0,0 +1,146 @@ +use super::super::trunk_id_selector; +use crate::config::{CrossOrigin, RtcBuild}; +use crate::pipelines::rust::{IntegrityOutput, RustAppType}; +use crate::processing::integrity::OutputDigest; +use nipper::Document; +use std::collections::HashMap; +use std::sync::Arc; + +/// The output of a cargo build pipeline. +pub struct RustAppOutput { + /// The runtime build config. + pub cfg: Arc, + /// The ID of this pipeline. + pub id: Option, + /// The filename of the generated JS loader file written to the dist dir. + pub js_output: String, + /// The filename of the generated WASM file written to the dist dir. + pub wasm_output: String, + /// The filename of the generated .ts file written to the dist dir. + pub ts_output: Option, + /// The filename of the generated loader shim script for web workers written to the dist dir. + pub loader_shim_output: Option, + /// Is this module main or a worker. + pub r#type: RustAppType, + /// The cross-origin setting for loading the resources + pub cross_origin: CrossOrigin, + /// The integrity and digest of the output, ignored in case of [`super::IntegrityType::None`] + pub integrity: IntegrityOutput, + /// The output digests for the discovered snippets + pub snippet_integrities: HashMap, + /// Import functions exported from Rust into JavaScript + pub import_bindings: bool, + /// The name of the WASM bindings import + pub import_bindings_name: Option, +} + +pub fn pattern_evaluate(template: &str, params: &HashMap) -> String { + let mut result = template.to_string(); + for (k, v) in params.iter() { + let pattern = format!("{{{}}}", k.as_str()); + if let Some(file_path) = v.strip_prefix('@') { + if let Ok(contents) = std::fs::read_to_string(file_path) { + result = str::replace(result.as_str(), &pattern, contents.as_str()); + } + } else { + result = str::replace(result.as_str(), &pattern, v); + } + } + result +} + +impl RustAppOutput { + pub async fn finalize(self, dom: &mut Document) -> anyhow::Result<()> { + if self.r#type == RustAppType::Worker { + // Skip the script tag and preload links for workers, and remove the link tag only. + // Workers are initialized and managed by the app itself at runtime. + if let Some(id) = self.id { + dom.select(&trunk_id_selector(id)).remove(); + } + return Ok(()); + } + + if !self.cfg.inject_scripts { + // Configuration directed we do not inject any scripts. + return Ok(()); + } + + let (base, js, wasm, head, body) = ( + &self.cfg.public_url, + &self.js_output, + &self.wasm_output, + "html head", + "html body", + ); + let (pattern_script, pattern_preload) = + (&self.cfg.pattern_script, &self.cfg.pattern_preload); + let mut params: HashMap = match &self.cfg.pattern_params { + Some(x) => x.clone(), + None => HashMap::new(), + }; + params.insert("base".to_owned(), base.clone()); + params.insert("js".to_owned(), js.clone()); + params.insert("wasm".to_owned(), wasm.clone()); + params.insert("crossorigin".to_owned(), self.cross_origin.to_string()); + + let preload = match pattern_preload { + Some(pattern) => pattern_evaluate(pattern, ¶ms), + None => { + format!( + r#" + +"#, + cross_origin = self.cross_origin, + wasm_integrity = self.integrity.wasm.make_attribute(), + js_integrity = self.integrity.js.make_attribute(), + ) + } + }; + dom.select(head).append_html(preload); + + for (name, integrity) in self.snippet_integrities { + if let Some(integrity) = integrity.to_integrity_value() { + let preload = format!( + r#" +"#, + cross_origin = self.cross_origin, + ); + dom.select(head).append_html(preload); + } + } + + let script = match pattern_script { + Some(pattern) => pattern_evaluate(pattern, ¶ms), + None => { + let (import, bind) = match self.import_bindings { + true => ( + ", * as bindings", + format!( + r#" +window.{bindings} = bindings; +"#, + bindings = self + .import_bindings_name + .as_deref() + .unwrap_or("wasmBindings") + ), + ), + false => ("", String::new()), + }; + format!( + r#" +"#, + ) + } + }; + + match self.id { + Some(id) => dom.select(&trunk_id_selector(id)).replace_with_html(script), + None => dom.select(body).append_html(script), + } + Ok(()) + } +} diff --git a/src/pipelines/sass.rs b/src/pipelines/sass.rs index 094ccc20..d462db59 100644 --- a/src/pipelines/sass.rs +++ b/src/pipelines/sass.rs @@ -1,18 +1,22 @@ //! Sass/Scss asset pipeline. +use super::{ + AssetFile, AttrWriter, Attrs, TrunkAssetPipelineOutput, ATTR_HREF, ATTR_INLINE, ATTR_INTEGRITY, +}; +use crate::{ + common, + config::RtcBuild, + processing::integrity::{IntegrityType, OutputDigest}, + tools::{self, Application}, +}; +use anyhow::{ensure, Context, Result}; +use nipper::Document; use std::path::PathBuf; +use std::str::FromStr; use std::sync::Arc; - -use anyhow::{Context, Result}; -use nipper::Document; use tokio::fs; use tokio::task::JoinHandle; -use super::{AssetFile, Attrs, TrunkAssetPipelineOutput, ATTR_HREF, ATTR_INLINE}; -use crate::common; -use crate::config::RtcBuild; -use crate::tools::{self, Application}; - /// A sass/scss asset pipeline. pub struct Sass { /// The ID of this pipeline's source HTML element. @@ -23,6 +27,10 @@ pub struct Sass { asset: AssetFile, /// If the specified SASS/SCSS file should be inlined. use_inline: bool, + /// E.g. `disabled`, `id="..."` + other_attrs: Attrs, + /// The required integrity setting + integrity: IntegrityType, } impl Sass { @@ -43,11 +51,20 @@ impl Sass { path.extend(href_attr.split('/')); let asset = AssetFile::new(&html_dir, path).await?; let use_inline = attrs.get(ATTR_INLINE).is_some(); + + let integrity = attrs + .get(ATTR_INTEGRITY) + .map(|value| IntegrityType::from_str(value)) + .transpose()? + .unwrap_or_default(); + Ok(Self { id, cfg, asset, use_inline, + other_attrs: attrs, + integrity, }) } @@ -60,29 +77,41 @@ impl Sass { /// Run this pipeline. #[tracing::instrument(level = "trace", skip(self))] async fn run(self) -> Result { - // tracing::info!("downloading sass"); let version = self.cfg.tools.sass.as_deref(); - let sass = tools::get(Application::Sass, version, &self.cfg.root_certificate, self.cfg.accept_invalid_certs.unwrap_or(false)).await?; - - // Compile the target SASS/SCSS file. - let style = if self.cfg.release { - "compressed" - } else { - "expanded" - }; - let path_str = dunce::simplified(&self.asset.path).display().to_string(); - let file_name = format!("{}.css", &self.asset.file_stem.to_string_lossy()); - let file_path = dunce::simplified(&self.cfg.staging_dist.join(&file_name)) - .display() - .to_string(); - let args = &["--no-source-map", "-s", style, &path_str, &file_path]; - - let rel_path = crate::common::strip_prefix(&self.asset.path); + let sass = tools::get(Application::Sass, version, self.cfg.offline, &self.cfg.root_certificate, self.cfg.accept_invalid_certs.unwrap_or(false)).await?; + + let source_path_str = dunce::simplified(&self.asset.path).display().to_string(); + let source_test = common::path_exists_and(&source_path_str, |m| m.is_file()).await; + ensure!( + source_test.ok() == Some(true), + "SASS source path '{source_path_str}' does not exist / is not a file" + ); + + let temp_target_file_name = format!("{}.css", &self.asset.file_stem.to_string_lossy()); + let temp_target_file_path = + dunce::simplified(&self.cfg.staging_dist.join(&temp_target_file_name)) + .display() + .to_string(); + + let args = &[ + "--no-source-map", + "--style", + match &self.cfg.release { + true => "compressed", + false => "expanded", + }, + &source_path_str, + &temp_target_file_path, + ]; + + let rel_path = common::strip_prefix(&self.asset.path); tracing::info!(path = ?rel_path, "compiling sass/scss"); common::run_command(Application::Sass.name(), &sass, args).await?; - let css = fs::read_to_string(&file_path).await?; - fs::remove_file(&file_path).await?; + let css = fs::read_to_string(&temp_target_file_path) + .await + .with_context(|| format!("error reading CSS result file '{temp_target_file_path}'"))?; + fs::remove_file(&temp_target_file_path).await?; // Check if the specified SASS/SCSS file should be inlined. let css_ref = if self.use_inline { @@ -96,16 +125,21 @@ impl Sass { .cfg .filehash .then(|| format!("{}-{:x}.css", &self.asset.file_stem.to_string_lossy(), hash)) - .unwrap_or(file_name); + .unwrap_or(temp_target_file_name); let file_path = self.cfg.staging_dist.join(&file_name); + let integrity = OutputDigest::generate_from(self.integrity, css.as_bytes()); + // Write the generated CSS to the filesystem. - fs::write(&file_path, css) - .await - .context("error writing SASS pipeline output")?; + fs::write(&file_path, css).await.with_context(|| { + format!( + "error writing SASS pipeline output file '{}'", + file_path.display() + ) + })?; // Generate a hashed reference to the new CSS file. - CssRef::File(file_name) + CssRef::File(file_name, integrity) }; tracing::info!(path = ?rel_path, "finished compiling sass/scss"); @@ -113,6 +147,7 @@ impl Sass { cfg: self.cfg.clone(), id: self.id, css_ref, + attrs: self.other_attrs, })) } } @@ -125,6 +160,8 @@ pub struct SassOutput { pub id: usize, /// Data on the finalized output file. pub css_ref: CssRef, + /// The other attributes copied over from the original. + pub attrs: Attrs, } /// The resulting CSS of the SASS/SCSS compilation. @@ -132,19 +169,26 @@ pub enum CssRef { /// CSS to be inlined (for `data-inline`). Inline(String), /// A hashed file reference to a CSS file (default). - File(String), + File(String, OutputDigest), } impl SassOutput { pub async fn finalize(self, dom: &mut Document) -> Result<()> { let html = match self.css_ref { // Insert the inlined CSS into a `"#, css), + CssRef::Inline(css) => format!( + r#""#, + attrs = AttrWriter::new(&self.attrs, AttrWriter::EXCLUDE_CSS_INLINE) + ), // Link to the CSS file. - CssRef::File(file) => { + CssRef::File(file, integrity) => { + let mut attrs = self.attrs.clone(); + integrity.insert_into(&mut attrs); + format!( - r#""#, + r#""#, base = &self.cfg.public_url, + attrs = AttrWriter::new(&attrs, AttrWriter::EXCLUDE_CSS_LINK) ) } }; diff --git a/src/pipelines/tailwind_css.rs b/src/pipelines/tailwind_css.rs index 98a6346d..b7ea51a4 100644 --- a/src/pipelines/tailwind_css.rs +++ b/src/pipelines/tailwind_css.rs @@ -1,6 +1,7 @@ //! Tailwind CSS asset pipeline. use std::path::PathBuf; +use std::str::FromStr; use std::sync::Arc; use anyhow::{Context, Result}; @@ -8,9 +9,12 @@ use nipper::Document; use tokio::fs; use tokio::task::JoinHandle; -use super::{AssetFile, Attrs, TrunkAssetPipelineOutput, ATTR_HREF, ATTR_INLINE}; +use super::{ + AssetFile, AttrWriter, Attrs, TrunkAssetPipelineOutput, ATTR_HREF, ATTR_INLINE, ATTR_INTEGRITY, +}; use crate::common; use crate::config::RtcBuild; +use crate::processing::integrity::{IntegrityType, OutputDigest}; use crate::tools::{self, Application}; /// A tailwind css asset pipeline. @@ -23,6 +27,10 @@ pub struct TailwindCss { asset: AssetFile, /// If the specified tailwind css file should be inlined. use_inline: bool, + /// E.g. `disabled`, `id="..."` + attrs: Attrs, + /// The required integrity setting + integrity: IntegrityType, } impl TailwindCss { @@ -42,11 +50,20 @@ impl TailwindCss { path.extend(href_attr.split('/')); let asset = AssetFile::new(&html_dir, path).await?; let use_inline = attrs.get(ATTR_INLINE).is_some(); + + let integrity = attrs + .get(ATTR_INTEGRITY) + .map(|value| IntegrityType::from_str(value)) + .transpose()? + .unwrap_or_default(); + Ok(Self { id, cfg, asset, use_inline, + integrity, + attrs, }) } @@ -60,7 +77,7 @@ impl TailwindCss { #[tracing::instrument(level = "trace", skip(self))] async fn run(self) -> Result { let version = self.cfg.tools.tailwindcss.as_deref(); - let tailwind = tools::get(Application::TailwindCss, version, &self.cfg.root_certificate, self.cfg.accept_invalid_certs.unwrap_or(false)).await?; + let tailwind = tools::get(Application::TailwindCss, version, self.cfg.offline, &self.cfg.root_certificate, self.cfg.accept_invalid_certs.unwrap_or(false)).await?; // Compile the target tailwind css file. let style = if self.cfg.release { "--minify" } else { "" }; @@ -93,13 +110,15 @@ impl TailwindCss { .unwrap_or(file_name); let file_path = self.cfg.staging_dist.join(&file_name); + let integrity = OutputDigest::generate_from(self.integrity, css.as_bytes()); + // Write the generated CSS to the filesystem. fs::write(&file_path, css) .await .context("error writing tailwind css pipeline output")?; // Generate a hashed reference to the new CSS file. - CssRef::File(file_name) + CssRef::File(file_name, integrity) }; tracing::info!(path = ?rel_path, "finished compiling tailwind css"); @@ -107,6 +126,7 @@ impl TailwindCss { cfg: self.cfg.clone(), id: self.id, css_ref, + attrs: self.attrs, })) } } @@ -119,6 +139,8 @@ pub struct TailwindCssOutput { pub id: usize, /// Data on the finalized output file. pub css_ref: CssRef, + /// The other attributes copied over from the original. + pub attrs: Attrs, } /// The resulting CSS of the Tailwind CSS compilation. @@ -126,19 +148,26 @@ pub enum CssRef { /// CSS to be inlined (for `data-inline`). Inline(String), /// A hashed file reference to a CSS file (default). - File(String), + File(String, OutputDigest), } impl TailwindCssOutput { pub async fn finalize(self, dom: &mut Document) -> Result<()> { let html = match self.css_ref { // Insert the inlined CSS into a `"#, css), + CssRef::Inline(css) => format!( + r#""#, + attrs = AttrWriter::new(&self.attrs, AttrWriter::EXCLUDE_CSS_INLINE) + ), // Link to the CSS file. - CssRef::File(file) => { + CssRef::File(file, integrity) => { + let mut attrs = self.attrs.clone(); + integrity.insert_into(&mut attrs); + format!( - r#""#, + r#""#, base = &self.cfg.public_url, + attrs = AttrWriter::new(&attrs, AttrWriter::EXCLUDE_CSS_LINK) ) } }; diff --git a/src/processing/integrity.rs b/src/processing/integrity.rs new file mode 100644 index 00000000..f8ec652e --- /dev/null +++ b/src/processing/integrity.rs @@ -0,0 +1,121 @@ +//! Integrity processing + +use base64::{display::Base64Display, engine::general_purpose::URL_SAFE}; +use sha2::{Digest, Sha256, Sha384, Sha512}; +use std::collections::HashMap; +use std::convert::Infallible; +use std::fmt::{Display, Formatter}; +use std::str::FromStr; + +/// Integrity type for subresource protection +#[derive(Copy, Clone, Eq, PartialEq, Hash, Debug, Default)] +pub enum IntegrityType { + None, + Sha256, + #[default] + Sha384, + Sha512, +} + +impl FromStr for IntegrityType { + type Err = IntegrityTypeParseError; + + fn from_str(s: &str) -> std::result::Result { + Ok(match s { + "" => Default::default(), + "none" => Self::None, + "sha256" => Self::Sha256, + "sha384" => Self::Sha384, + "sha512" => Self::Sha512, + _ => return Err(IntegrityTypeParseError::InvalidValue), + }) + } +} + +impl Display for IntegrityType { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match self { + Self::None => write!(f, "none"), + Self::Sha256 => write!(f, "sha256"), + Self::Sha384 => write!(f, "sha384"), + Self::Sha512 => write!(f, "sha512"), + } + } +} + +#[derive(Debug, thiserror::Error)] +pub enum IntegrityTypeParseError { + #[error("invalid value")] + InvalidValue, +} + +/// The digest of the output +#[derive(Debug)] +pub struct OutputDigest { + /// The digest algorithm + pub integrity: IntegrityType, + /// The raw hash/digest value + pub hash: Vec, +} + +impl Default for OutputDigest { + fn default() -> Self { + Self { + integrity: IntegrityType::None, + hash: vec![], + } + } +} + +impl OutputDigest { + /// Turn into a SRI attribute with can be appended to a string. + pub fn make_attribute(&self) -> String { + self.to_integrity_value() + .map(|value| { + // format of an attribute, including the leading space + format!(r#" integrity="{value}""#) + }) + .unwrap_or_default() + } + + /// Turn into the value for an SRI attribute + pub fn to_integrity_value(&self) -> Option { + match self.integrity { + IntegrityType::None => None, + integrity => Some(format!( + "{integrity}-{hash}", + hash = Base64Display::new(&self.hash, &URL_SAFE) + )), + } + } + + /// Insert as an SRI attribute into a an [`Attrs`] instance. + pub fn insert_into(&self, attrs: &mut HashMap) { + if let Some(value) = self.to_integrity_value() { + attrs.insert("integrity".to_string(), value.to_string()); + } + } + + /// Generate from input data + pub fn generate(integrity: IntegrityType, f: F) -> Result + where + F: FnOnce() -> Result, + T: AsRef<[u8]>, + { + let hash = match integrity { + IntegrityType::None => vec![], + IntegrityType::Sha256 => Vec::from_iter(Sha256::digest(f()?)), + IntegrityType::Sha384 => Vec::from_iter(Sha384::digest(f()?)), + IntegrityType::Sha512 => Vec::from_iter(Sha512::digest(f()?)), + }; + + Ok(Self { integrity, hash }) + } + + /// Generate from existing input data + pub fn generate_from(integrity: IntegrityType, data: impl AsRef<[u8]>) -> Self { + Self::generate::<_, _, Infallible>(integrity, || Ok(data)) + // we can safely unwrap, as we know it's infallible + .unwrap() + } +} diff --git a/src/processing/mod.rs b/src/processing/mod.rs new file mode 100644 index 00000000..5936ef09 --- /dev/null +++ b/src/processing/mod.rs @@ -0,0 +1,3 @@ +//! Functionality for processing + +pub mod integrity; diff --git a/src/serve.rs b/src/serve.rs index fa41e1c2..da3fba0a 100644 --- a/src/serve.rs +++ b/src/serve.rs @@ -1,24 +1,30 @@ -use std::net::{IpAddr, Ipv4Addr}; -use std::path::PathBuf; -use std::sync::Arc; - +use crate::common::{LOCAL, NETWORK, SERVER}; +use crate::config::RtcServe; +use crate::proxy::{ProxyHandlerHttp, ProxyHandlerWebSocket}; +use crate::watch::WatchSystem; +use crate::ws; use anyhow::{Context, Result}; -use axum::body::{self, Body}; -use axum::extract::ws::{WebSocket, WebSocketUpgrade}; -use axum::http::StatusCode; +use axum::body::{self, Body, Bytes}; +use axum::extract::ws::WebSocketUpgrade; +use axum::http::header::{HeaderName, CONTENT_LENGTH, CONTENT_TYPE, HOST}; +use axum::http::response::Parts; +use axum::http::{HeaderValue, Request, StatusCode}; +use axum::middleware::Next; use axum::response::Response; use axum::routing::{get, get_service, Router}; -use axum::Server; -use tokio::sync::broadcast; +use axum_server::tls_rustls::RustlsConfig; +use axum_server::Handle; +use std::collections::HashMap; +use std::net::{IpAddr, Ipv4Addr, SocketAddr}; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{broadcast, watch}; use tokio::task::JoinHandle; use tower_http::services::{ServeDir, ServeFile}; +use tower_http::set_header::SetResponseHeaderLayer; use tower_http::trace::TraceLayer; -use crate::common::{LOCAL, NETWORK, SERVER}; -use crate::config::RtcServe; -use crate::proxy::{ProxyHandlerHttp, ProxyHandlerWebSocket}; -use crate::watch::WatchSystem; - const INDEX_HTML: &str = "index.html"; /// A system encapsulating a build & watch system, responsible for serving generated content. @@ -29,29 +35,31 @@ pub struct ServeSystem { shutdown_tx: broadcast::Sender<()>, // N.B. we use a broadcast channel here because a watch channel triggers a // false positive on the first read of channel - build_done_chan: broadcast::Sender<()>, + ws_state: watch::Receiver, } impl ServeSystem { /// Construct a new instance. pub async fn new(cfg: Arc, shutdown: broadcast::Sender<()>) -> Result { - let (build_done_chan, _) = broadcast::channel(8); + let (ws_state_tx, ws_state) = watch::channel(ws::State::default()); let watch = WatchSystem::new( cfg.watch.clone(), shutdown.clone(), - Some(build_done_chan.clone()), + Some(ws_state_tx), + cfg.ws_protocol, ) .await?; + let prefix = if cfg.tls.is_some() { "https" } else { "http" }; let http_addr = format!( - "http://{}:{}{}", - cfg.address, cfg.port, &cfg.watch.build.public_url + "{}://{}:{}{}", + prefix, cfg.address, cfg.port, &cfg.watch.build.public_url ); Ok(Self { cfg, watch, http_addr, shutdown_tx: shutdown, - build_done_chan, + ws_state, }) } @@ -64,8 +72,9 @@ impl ServeSystem { let server_handle = Self::spawn_server( self.cfg.clone(), self.shutdown_tx.subscribe(), - self.build_done_chan, - )?; + self.ws_state, + ) + .await?; // Open the browser. if self.cfg.open { @@ -84,18 +93,11 @@ impl ServeSystem { } #[tracing::instrument(level = "trace", skip(cfg, shutdown_rx))] - fn spawn_server( + async fn spawn_server( cfg: Arc, - mut shutdown_rx: broadcast::Receiver<()>, - build_done_chan: broadcast::Sender<()>, + shutdown_rx: broadcast::Receiver<()>, + ws_state: watch::Receiver, ) -> Result> { - // Build a shutdown signal for the warp server. - let shutdown_fut = async move { - // Any event on this channel, even a drop, should trigger shutdown. - let _res = shutdown_rx.recv().await; - tracing::debug!("server is shutting down"); - }; - // Build the proxy client. let client = reqwest::ClientBuilder::new() .http1_only() @@ -115,14 +117,14 @@ impl ServeSystem { client, insecure_client, &cfg, - build_done_chan, + ws_state, )); - let router = router(state, cfg.clone()); + let router = router(state, cfg.clone())?; let addr = (cfg.address, cfg.port).into(); - let server = Server::bind(&addr) - .serve(router.into_make_service()) - .with_graceful_shutdown(shutdown_fut); + let server = run_server(addr, cfg.tls.clone(), router, shutdown_rx); + + let prefix = if cfg.tls.is_some() { "https" } else { "http" }; if addr.ip().is_unspecified() { let addresses = local_ip_address::list_afinet_netifas() .map(|addrs| { @@ -136,17 +138,18 @@ impl ServeSystem { }) .unwrap_or_else(|_| vec![Ipv4Addr::LOCALHOST]); tracing::info!( - "{} server listening at:\n{}", + "{}server listening at:\n{}", SERVER, addresses .iter() .map(|address| format!( - " {} http://{}:{}", + " {}{}://{}:{}", if address.is_loopback() { LOCAL } else { NETWORK }, + prefix, address, cfg.port )) @@ -154,7 +157,7 @@ impl ServeSystem { .join("\n") ); } else { - tracing::info!("{} server listening at http://{}", SERVER, addr); + tracing::info!("{}server listening at {}://{}", SERVER, prefix, addr); } // Block this routine on the server's completion. Ok(tokio::spawn(async move { @@ -165,6 +168,41 @@ impl ServeSystem { } } +async fn run_server( + addr: SocketAddr, + tls: Option, + router: Router, + mut shutdown_rx: broadcast::Receiver<()>, +) -> Result<()> { + // Build a shutdown signal for the axum server. + let shutdown_handle = Handle::new(); + + let shutdown = |handle: Handle| async move { + // Any event on this channel, even a drop, should trigger shutdown. + let _res = shutdown_rx.recv().await; + tracing::debug!("server is shutting down"); + handle.graceful_shutdown(Some(Duration::from_secs(0))); + }; + + tokio::spawn(shutdown(shutdown_handle.clone())); + match tls { + Some(tls_config) => { + axum_server::bind_rustls(addr, tls_config.clone()) + .handle(shutdown_handle) + .serve(router.into_make_service()) + .await + } + None => { + axum_server::bind(addr) + .handle(shutdown_handle) + .serve(router.into_make_service()) + .await + } + }?; + + Ok(()) +} + /// Server state. pub struct State { /// A client instance used by proxies. @@ -175,10 +213,12 @@ pub struct State { pub dist_dir: PathBuf, /// The public URL from which assets are being served. pub public_url: String, - /// The channel to receive build_done notifications on. - pub build_done_chan: broadcast::Sender<()>, + /// The channel for WS client messages. + pub ws_state: watch::Receiver, /// Whether to disable autoreload pub no_autoreload: bool, + /// Additional headers to add to responses. + pub headers: HashMap, } impl State { @@ -189,22 +229,23 @@ impl State { client: reqwest::Client, insecure_client: reqwest::Client, cfg: &RtcServe, - build_done_chan: broadcast::Sender<()>, + ws_state: watch::Receiver, ) -> Self { Self { client, insecure_client, dist_dir, public_url, - build_done_chan, + ws_state, no_autoreload: cfg.no_autoreload, + headers: cfg.headers.clone(), } } } /// Build the Trunk router, this includes that static file server, the WebSocket server, /// (for autoreload & HMR in the future), as well as any user-defined proxies. -fn router(state: Arc, cfg: Arc) -> Router { +fn router(state: Arc, cfg: Arc) -> Result { // Build static file server, middleware, error handler & WS route for reloads. let public_route = if state.public_url == "/" { &state.public_url @@ -215,33 +256,48 @@ fn router(state: Arc, cfg: Arc) -> Router { .unwrap_or(&state.public_url) }; + let mut serve_dir = if cfg.no_spa { + get_service(ServeDir::new(&state.dist_dir)) + } else { + get_service( + ServeDir::new(&state.dist_dir) + .fallback(ServeFile::new(state.dist_dir.join(INDEX_HTML))), + ) + }; + for (key, value) in &state.headers { + let name = HeaderName::from_bytes(key.as_bytes()) + .with_context(|| format!("invalid header {:?}", key))?; + let value: HeaderValue = value + .parse() + .with_context(|| format!("invalid header value {:?} for header {}", value, name))?; + serve_dir = serve_dir.layer(SetResponseHeaderLayer::overriding(name, value)) + } + let mut router = Router::new() .fallback_service( Router::new().nest_service( public_route, - get_service( - ServeDir::new(&state.dist_dir) - .fallback(ServeFile::new(state.dist_dir.join(INDEX_HTML))), - ) - .handle_error(|error| async move { - tracing::error!(?error, "failed serving static file"); - StatusCode::INTERNAL_SERVER_ERROR - }) - .layer(TraceLayer::new_for_http()), + get_service(serve_dir) + .handle_error(|error| async move { + tracing::error!(?error, "failed serving static file"); + StatusCode::INTERNAL_SERVER_ERROR + }) + .layer(TraceLayer::new_for_http()) + .layer(axum::middleware::from_fn(html_address_middleware)), ), ) .route( "/_trunk/ws", get( |ws: WebSocketUpgrade, state: axum::extract::State>| async move { - ws.on_upgrade(|socket| async move { handle_ws(socket, state.0).await }) + ws.on_upgrade(|socket| async move { ws::handle_ws(socket, state.0).await }) }, ), ) .with_state(state.clone()); tracing::info!( - "{} serving static assets at -> {}", + "{}serving static assets at -> {}", SERVER, state.public_url.as_str() ); @@ -252,7 +308,7 @@ fn router(state: Arc, cfg: Arc) -> Router { let handler = ProxyHandlerWebSocket::new(backend.clone(), cfg.proxy_rewrite.clone()); router = handler.clone().register(router); tracing::info!( - "{} proxying websocket {} -> {}", + "{}proxying websocket {} -> {}", SERVER, handler.path(), &backend @@ -266,7 +322,7 @@ fn router(state: Arc, cfg: Arc) -> Router { let handler = ProxyHandlerHttp::new(client, backend.clone(), cfg.proxy_rewrite.clone()); router = handler.clone().register(router); - tracing::info!("{} proxying {} -> {}", SERVER, handler.path(), &backend); + tracing::info!("{}proxying {} -> {}", SERVER, handler.path(), &backend); } } else if let Some(proxies) = &cfg.proxies { for proxy in proxies.iter() { @@ -275,7 +331,7 @@ fn router(state: Arc, cfg: Arc) -> Router { ProxyHandlerWebSocket::new(proxy.backend.clone(), proxy.rewrite.clone()); router = handler.clone().register(router); tracing::info!( - "{} proxying websocket {} -> {}", + "{}proxying websocket {} -> {}", SERVER, handler.path(), &proxy.backend @@ -291,7 +347,7 @@ fn router(state: Arc, cfg: Arc) -> Router { ProxyHandlerHttp::new(client, proxy.backend.clone(), proxy.rewrite.clone()); router = handler.clone().register(router); tracing::info!( - "{} proxying {} -> {}", + "{}proxying {} -> {}", SERVER, handler.path(), &proxy.backend @@ -300,24 +356,42 @@ fn router(state: Arc, cfg: Arc) -> Router { } } - router + Ok(router) } -async fn handle_ws(mut ws: WebSocket, state: Arc) { - let mut rx = state.build_done_chan.subscribe(); - tracing::debug!("autoreload websocket opened"); - while tokio::select! { - _ = ws.recv() => { - tracing::debug!("autoreload websocket closed"); - return - } - build_done = rx.recv() => build_done.is_ok(), - } { - let ws_send = ws.send(axum::extract::ws::Message::Text( - r#"{"reload": true}"#.to_owned(), - )); - if ws_send.await.is_err() { - break; +async fn html_address_middleware( + request: Request, + next: Next, +) -> (Parts, Bytes) { + let uri = request.headers().get(HOST).cloned(); + let response = next.run(request).await; + let (parts, body) = response.into_parts(); + + match hyper::body::to_bytes(body).await { + Err(_) => (parts, Bytes::default()), + Ok(bytes) => { + let (mut parts, mut bytes) = (parts, bytes); + + // turn into a string literal, or replace with "current host" on the client side + let uri = uri + .and_then(|uri| uri.to_str().map(|s| format!("'{}'", s)).ok()) + .unwrap_or_else(|| "window.location.host".into()); + + if parts + .headers + .get(CONTENT_TYPE) + .map(|t| t == "text/html") + .unwrap_or(false) + { + if let Ok(data_str) = std::str::from_utf8(&bytes) { + let data_str = data_str.replace("'{{__TRUNK_ADDRESS__}}'", &uri); + let bytes_vec = data_str.as_bytes().to_vec(); + parts.headers.insert(CONTENT_LENGTH, bytes_vec.len().into()); + bytes = Bytes::from(bytes_vec); + } + } + + (parts, bytes) } } } diff --git a/src/tools.rs b/src/tools.rs index fc0ca429..7f724e74 100644 --- a/src/tools.rs +++ b/src/tools.rs @@ -4,7 +4,7 @@ use std::collections::HashMap; use std::path::PathBuf; -use anyhow::{bail, ensure, Context, Result, anyhow}; +use anyhow::{anyhow, bail, ensure, Context, Result}; use directories::ProjectDirs; use futures_util::stream::StreamExt; use once_cell::sync::Lazy; @@ -14,10 +14,10 @@ use tokio::process::Command; use tokio::sync::{Mutex, OnceCell}; use self::archive::Archive; -use crate::common::is_executable; +use crate::common::{is_executable, path_exists, path_exists_and}; /// The application to locate and eventually download when calling [`get`]. -#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, strum::EnumIter)] pub enum Application { /// sass for generating css Sass, @@ -41,7 +41,7 @@ impl Application { } /// Path of the executable within the downloaded archive. - fn path(&self) -> &str { + pub(crate) fn path(&self) -> &str { if cfg!(target_os = "windows") { match self { Self::Sass => "sass.bat", @@ -60,7 +60,7 @@ impl Application { } /// Additional files included in the archive that are required to run the main binary. - fn extra_paths(&self) -> &[&str] { + pub(crate) fn extra_paths(&self) -> &[&str] { match self { Self::Sass => { if cfg!(target_os = "windows") { @@ -82,17 +82,17 @@ impl Application { } /// Default version to use if not set by the user. - fn default_version(&self) -> &str { + pub(crate) fn default_version(&self) -> &str { match self { - Self::Sass => "1.63.6", - Self::TailwindCss => "3.3.2", - Self::WasmBindgen => "0.2.87", - Self::WasmOpt => "version_113", + Self::Sass => "1.69.5", + Self::TailwindCss => "3.3.5", + Self::WasmBindgen => "0.2.88", + Self::WasmOpt => "version_116", } } /// Direct URL to the release of an application for download. - fn url(&self, version: &str) -> Result { + pub(crate) fn url(&self, version: &str) -> Result { let target_os = if cfg!(target_os = "windows") { "windows" } else if cfg!(target_os = "macos") { @@ -113,10 +113,10 @@ impl Application { Ok(match self { Self::Sass => match (target_os, target_arch) { - ("windows", "x86_64") => format!("https://github.com/sass/dart-sass/releases/download/{version}/dart-sass-{version}-windows-x64.zip"), - ("macos" | "linux", "x86_64") => format!("https://github.com/sass/dart-sass/releases/download/{version}/dart-sass-{version}-{target_os}-x64.tar.gz"), - ("macos" | "linux", "aarch64") => format!("https://github.com/sass/dart-sass/releases/download/{version}/dart-sass-{version}-{target_os}-arm64.tar.gz"), - _ => bail!("Unable to download Sass for {target_os} {target_arch}") + ("windows", "x86_64") => format!("https://github.com/sass/dart-sass/releases/download/{version}/dart-sass-{version}-windows-x64.zip"), + ("macos" | "linux", "x86_64") => format!("https://github.com/sass/dart-sass/releases/download/{version}/dart-sass-{version}-{target_os}-x64.tar.gz"), + ("macos" | "linux", "aarch64") => format!("https://github.com/sass/dart-sass/releases/download/{version}/dart-sass-{version}-{target_os}-arm64.tar.gz"), + _ => bail!("Unable to download Sass for {target_os} {target_arch}") }, Self::TailwindCss => match (target_os, target_arch) { @@ -127,17 +127,17 @@ impl Application { }, Self::WasmBindgen => match (target_os, target_arch) { - ("windows", "x86_64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-x86_64-pc-windows-msvc.tar.gz"), - ("macos", "x86_64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-x86_64-apple-darwin.tar.gz"), - ("macos", "aarch64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-aarch64-apple-darwin.tar.gz"), - ("linux", "x86_64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-x86_64-unknown-linux-musl.tar.gz"), - ("linux", "aarch64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-aarch64-unknown-linux-gnu.tar.gz"), - _ => bail!("Unable to download wasm-bindgen for {target_os} {target_arch}") + ("windows", "x86_64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-x86_64-pc-windows-msvc.tar.gz"), + ("macos", "x86_64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-x86_64-apple-darwin.tar.gz"), + ("macos", "aarch64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-aarch64-apple-darwin.tar.gz"), + ("linux", "x86_64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-x86_64-unknown-linux-musl.tar.gz"), + ("linux", "aarch64") => format!("https://github.com/rustwasm/wasm-bindgen/releases/download/{version}/wasm-bindgen-{version}-aarch64-unknown-linux-gnu.tar.gz"), + _ => bail!("Unable to download wasm-bindgen for {target_os} {target_arch}") }, Self::WasmOpt => match (target_os, target_arch) { - ("macos", "aarch64") => format!("https://github.com/WebAssembly/binaryen/releases/download/{version}/binaryen-{version}-arm64-macos.tar.gz"), - _ => format!("https://github.com/WebAssembly/binaryen/releases/download/{version}/binaryen-{version}-{target_arch}-{target_os}.tar.gz") + ("macos", "aarch64") => format!("https://github.com/WebAssembly/binaryen/releases/download/{version}/binaryen-{version}-arm64-macos.tar.gz"), + _ => format!("https://github.com/WebAssembly/binaryen/releases/download/{version}/binaryen-{version}-{target_arch}-{target_os}.tar.gz") } }) } @@ -153,11 +153,11 @@ impl Application { } /// Format the output of version checking the app. - fn format_version_output(&self, text: &str) -> Result { + pub(crate) fn format_version_output(&self, text: &str) -> Result { let text = text.trim(); let formatted_version = match self { Application::Sass => text - .lines() + .split_whitespace() .next() .with_context(|| format!("missing or malformed version output: {}", text))? .to_owned(), @@ -211,10 +211,7 @@ impl AppCache { root_certificate: &Option, accept_invalid_certs: bool, ) -> Result<()> { - let cached = self - .0 - .entry((app, version.to_owned())) - .or_insert_with(OnceCell::new); + let cached = self.0.entry((app, version.to_owned())).or_default(); cached .get_or_try_init(|| async move { @@ -239,12 +236,16 @@ impl AppCache { /// Locate the given application and download it if missing. #[tracing::instrument(level = "trace")] -pub async fn get(app: Application, version: Option<&str>, root_certificate: &Option, accept_invalid_certs: bool) -> Result { +pub async fn get(app: Application, version: Option<&str>, offline: bool, root_certificate: &Option, accept_invalid_certs: bool) -> Result { if let Some((path, version)) = find_system(app, version).await { tracing::info!(app = %app.name(), %version, "using system installed binary"); return Ok(path); } + if offline { + return Err(anyhow!("couldn't find application {}", &app.name())); + } + let cache_dir = cache_dir().await?; let version = version.unwrap_or_else(|| app.default_version()); let app_dir = cache_dir.join(format!("{}-{}", app.name(), version)); @@ -264,7 +265,7 @@ pub async fn get(app: Application, version: Option<&str>, root_certificate: &Opt /// Try to find a globally system installed version of the application and ensure it is the needed /// release version. #[tracing::instrument(level = "trace")] -async fn find_system(app: Application, version: Option<&str>) -> Option<(PathBuf, String)> { +pub async fn find_system(app: Application, version: Option<&str>) -> Option<(PathBuf, String)> { let result = || async { let path = which::which(app.name())?; let output = Command::new(&path).arg(app.version_test()).output().await?; @@ -330,11 +331,12 @@ async fn download(app: Application, version: &str, root_certificate: &Option Result<()> { +async fn install(app: Application, archive_file: File, target_directory: PathBuf) -> Result<()> { tracing::info!("installing {}", app.name()); let archive_file = archive_file.into_std().await; + let target_directory_clone = target_directory.clone(); tokio::task::spawn_blocking(move || { let mut archive = if app == Application::Sass && cfg!(target_os = "windows") { Archive::new_zip(archive_file)? @@ -343,12 +345,12 @@ async fn install(app: Application, archive_file: File, target: PathBuf) -> Resul } else { Archive::new_tar_gz(archive_file) }; - archive.extract_file(app.path(), &target)?; + archive.extract_file(app.path(), &target_directory)?; for path in app.extra_paths() { // After extracting one file the archive must be reset. archive = archive.reset()?; - if archive.extract_file(path, &target).is_err() { + if archive.extract_file(path, &target_directory).is_err() { tracing::warn!( "attempted to extract '{}' from {:?} archive, but it is not present, this \ could be due to version updates", @@ -358,9 +360,32 @@ async fn install(app: Application, archive_file: File, target: PathBuf) -> Resul } } - Ok(()) + Result::<()>::Ok(()) }) - .await? + .await + .context("Unable to join on spawn_blocking")? + .context("Could not extract files")?; + + let main_executable = target_directory_clone.join(app.path()); + let test = path_exists(&main_executable).await; + ensure!( + test.ok() == Some(true), + "Extracted application binary {main_executable:?} could not be found." + ); + + let test = path_exists_and(&main_executable, |m| m.is_file()).await; + ensure!( + test.ok() == Some(true), + "Extracted application binary {main_executable:?} is not a file" + ); + + let test = is_executable(&main_executable).await; + ensure!( + test.ok() == Some(true), + "Extracted application binary {main_executable:?} is not executable." + ); + + Ok(()) } /// Locate the cache dir for trunk and make sure it exists. @@ -400,6 +425,7 @@ async fn get_http_client(root_certificate: &Option, accept_invalid_cert } mod archive { + use std::fmt::Display; use std::fs::{self, File}; use std::io::{self, BufReader, BufWriter, Read, Seek}; use std::path::Path; @@ -430,47 +456,46 @@ mod archive { Self::None(file) } - pub fn extract_file(&mut self, file: &str, target: &Path) -> Result<()> { + pub fn extract_file(&mut self, file: &str, target_directory: &Path) -> Result<()> { match self { Self::TarGz(archive) => { let mut tar_file = find_tar_entry(archive, file)?.context("file not found in archive")?; - let mut out_file = extract_file(&mut tar_file, file, target)?; + let mut out_file = extract_file(&mut tar_file, file, target_directory)?; if let Ok(mode) = tar_file.header().mode() { - set_file_permissions(&mut out_file, mode)?; + set_file_permissions(&mut out_file, mode, file)?; } } Self::Zip(archive) => { let zip_index = find_zip_entry(archive, file)?.context("file not found in archive")?; let mut zip_file = archive.by_index(zip_index)?; - let mut out_file = extract_file(&mut zip_file, file, target)?; + let mut out_file = extract_file(&mut zip_file, file, target_directory)?; if let Some(mode) = zip_file.unix_mode() { - set_file_permissions(&mut out_file, mode)?; + set_file_permissions(&mut out_file, mode, file)?; } } Self::None(in_file) => { - let create_dir_result = std::fs::create_dir(target); + let create_dir_result = std::fs::create_dir(target_directory); if let Err(e) = &create_dir_result { if e.kind() != std::io::ErrorKind::AlreadyExists { create_dir_result.context("failed to open file for")?; } } - let mut out_file_path = target.to_path_buf(); + let mut out_file_path = target_directory.to_path_buf(); out_file_path.push(file); let mut out_file = - File::create(out_file_path).context("failed to open binary to copy")?; + File::create(&out_file_path).context("failed to open binary to copy")?; { let mut reader = BufReader::new(in_file); let mut writer = BufWriter::new(&out_file); std::io::copy(&mut reader, &mut writer).context("failed to copy binary")?; } - set_file_permissions(&mut out_file, 0o755)?; // rwx for user, rx for group and - // other. + set_file_permissions(&mut out_file, 0o755, out_file_path.display())?; } } @@ -541,14 +566,15 @@ mod archive { Ok(None) } - fn extract_file(mut read: impl Read, file: &str, target: &Path) -> Result { - let out = target.join(file); + fn extract_file(mut read: impl Read, file: &str, target_directory: &Path) -> Result { + let out = target_directory.join(file); if let Some(parent) = out.parent() { fs::create_dir_all(parent).context("failed creating output directory")?; } - let mut out = File::create(target.join(file)).context("failed creating output file")?; + let mut out = + File::create(target_directory.join(file)).context("failed creating output file")?; io::copy(&mut read, &mut out) .context("failed copying over final output file from archive")?; @@ -556,12 +582,18 @@ mod archive { } /// Set the executable flag for a file. Only has an effect on UNIX platforms. - fn set_file_permissions(file: &mut File, mode: u32) -> Result<()> { + fn set_file_permissions( + file: &mut File, + mode: u32, + file_path_hint: impl Display, + ) -> Result<()> { #[cfg(unix)] { use std::fs::Permissions; use std::os::unix::fs::PermissionsExt; + tracing::debug!("Setting permission of '{file_path_hint}' to {mode:#o}"); + file.set_permissions(Permissions::from_mode(mode)) .context("failed setting file permissions")?; } @@ -647,6 +679,12 @@ mod tests { ); table_test_format_version!(sass_pre_compiled, Application::Sass, "1.37.5", "1.37.5"); + table_test_format_version!( + sass_pre_compiled_dart2js, + Application::Sass, + "1.37.5 compiled with dart2js 2.18.4", + "1.37.5" + ); table_test_format_version!( tailwindcss_pre_compiled, Application::TailwindCss, diff --git a/src/watch.rs b/src/watch.rs index 26a48d1c..a29b8b66 100644 --- a/src/watch.rs +++ b/src/watch.rs @@ -1,47 +1,81 @@ -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; - +use crate::build::{BuildResult, BuildSystem}; +use crate::config::{RtcWatch, WsProtocol}; +use crate::ws; use anyhow::{Context, Result}; use futures_util::stream::StreamExt; -use notify::event::ModifyKind; -use notify::{EventKind, RecommendedWatcher, RecursiveMode, Watcher}; +use notify::event::{MetadataKind, ModifyKind}; +use notify::{EventKind, PollWatcher, RecommendedWatcher, RecursiveMode, Watcher}; use notify_debouncer_full::{ - new_debouncer, DebounceEventResult, DebouncedEvent, Debouncer, FileIdMap, + new_debouncer_opt, DebounceEventResult, DebouncedEvent, Debouncer, FileIdMap, }; -use tokio::sync::{broadcast, mpsc}; +use parking_lot::MappedMutexGuard; +use std::fmt::Write; +use std::path::PathBuf; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::{broadcast, mpsc, watch, Mutex}; use tokio::time::Instant; use tokio_stream::wrappers::BroadcastStream; -use crate::build::BuildSystem; -use crate::config::RtcWatch; +pub enum FsDebouncer { + Default(Debouncer), + Polling(Debouncer), +} -/// The debouncer type used in this module. -type FsDebouncer = Debouncer; +impl FsDebouncer { + pub fn watcher(&mut self) -> &mut dyn Watcher { + match self { + Self::Default(deb) => deb.watcher(), + Self::Polling(deb) => deb.watcher(), + } + } + + pub fn cache(&mut self) -> MappedMutexGuard { + match self { + Self::Default(deb) => deb.cache(), + Self::Polling(deb) => deb.cache(), + } + } +} /// Blacklisted path segments which are ignored by the watcher by default. -const BLACKLIST: [&str; 1] = [".git"]; +const BLACKLIST: [&str; 2] = [".git", ".DS_Store"]; /// The duration of time to debounce FS events. const DEBOUNCE_DURATION: Duration = Duration::from_millis(25); /// The duration of time during which watcher events will be ignored following a build. +/// +/// There are various OS syscalls which can trigger FS changes, even though semantically +/// no changes were made. A notorious example which has plagued the trunk +/// watcher implementation is `std::fs::copy`, which will trigger watcher +/// changes indicating that file contents have been modified. +/// +/// Given the difficult nature of this issue, we opt for using a cooldown period. Any +/// changes events processed within the cooldown period following a build +/// will be ignored. const WATCHER_COOLDOWN: Duration = Duration::from_secs(1); /// A watch system wrapping a build system and a watcher. pub struct WatchSystem { /// The build system. - build: BuildSystem, + build: Arc>, /// The current vector of paths to be ignored. ignored_paths: Vec, /// A channel of FS watch events. watch_rx: mpsc::Receiver, /// A channel of new paths to ignore from the build system. - build_rx: mpsc::Receiver, + ignore_rx: mpsc::Receiver, + /// A sender to notify the end of a build. + build_tx: mpsc::Sender, + /// A channel to receive the end of a build. + build_rx: mpsc::Receiver, /// The watch system used for watching the filesystem. _debouncer: FsDebouncer, /// The application shutdown channel. shutdown: BroadcastStream<()>, - /// Channel that is sent on whenever a build completes. - build_done_tx: Option>, + /// Channel to communicate with the client socket + ws_state: Option>, + /// Timestamp the last build was started. + last_build_started: Instant, /// An instant used to track the last build time, used to implement the watcher cooldown /// to avoid infinite build loops. /// @@ -50,6 +84,12 @@ pub struct WatchSystem { /// build cooldown period ensures that no FS events are processed until at least a duration /// of `WATCHER_COOLDOWN` has elapsed since the last build. last_build_finished: Instant, + /// The timestamp of the last accepted change event. + last_change: Instant, + /// The cooldown for the watcher. [`None`] disables the cooldown. + watcher_cooldown: Option, + /// Don't send build errors to the frontend. + no_error_reporting: bool, } impl WatchSystem { @@ -57,33 +97,50 @@ impl WatchSystem { pub async fn new( cfg: Arc, shutdown: broadcast::Sender<()>, - build_done_tx: Option>, + ws_state: Option>, + ws_protocol: Option, ) -> Result { // Create a channel for being able to listen for new paths to ignore while running. let (watch_tx, watch_rx) = mpsc::channel(1); + let (ignore_tx, ignore_rx) = mpsc::channel(1); let (build_tx, build_rx) = mpsc::channel(1); // Build the watcher. - let _debouncer = build_watcher(watch_tx, cfg.paths.clone())?; + let _debouncer = build_watcher(watch_tx, cfg.paths.clone(), cfg.poll)?; + + // Cooldown + let watcher_cooldown = cfg.enable_cooldown.then_some(WATCHER_COOLDOWN); + tracing::debug!( + "Build cooldown: {:?}", + watcher_cooldown.map(humantime::Duration::from) + ); // Build dependencies. - let build = BuildSystem::new(cfg.build.clone(), Some(build_tx)).await?; + let build = Arc::new(Mutex::new( + BuildSystem::new(cfg.build.clone(), Some(ignore_tx), ws_protocol).await?, + )); Ok(Self { build, ignored_paths: cfg.ignored_paths.clone(), watch_rx, + ignore_rx, build_rx, + build_tx, _debouncer, shutdown: BroadcastStream::new(shutdown.subscribe()), - build_done_tx, + ws_state, + last_build_started: Instant::now(), last_build_finished: Instant::now(), + last_change: Instant::now(), + watcher_cooldown, + no_error_reporting: cfg.no_error_reporting, }) } /// Run a build. #[tracing::instrument(level = "trace", skip(self))] pub async fn build(&mut self) -> Result<()> { - self.build.build().await + self.build.lock().await.build().await } /// Run the watch system, responding to events and triggering builds. @@ -91,8 +148,9 @@ impl WatchSystem { pub async fn run(mut self) { loop { tokio::select! { - Some(ign) = self.build_rx.recv() => self.update_ignore_list(ign), + Some(ign) = self.ignore_rx.recv() => self.update_ignore_list(ign), Some(ev) = self.watch_rx.recv() => self.handle_watch_event(ev).await, + Some(build) = self.build_rx.recv() => self.build_complete(build).await, _ = self.shutdown.next() => break, // Any event, even a drop, will trigger shutdown. } } @@ -100,29 +158,113 @@ impl WatchSystem { tracing::debug!("watcher system has shut down"); } + #[tracing::instrument(level = "trace", skip(self))] + async fn build_complete(&mut self, build_result: Result<(), anyhow::Error>) { + tracing::debug!("Build reported completion"); + + // record last finish timestamp + self.last_build_finished = Instant::now(); + + if let Some(tx) = &mut self.ws_state { + match build_result { + Ok(()) => { + let _ = tx.send_replace(ws::State::Ok); + } + Err(err) => { + if !self.no_error_reporting { + let _ = tx.send_replace(ws::State::Failed { + reason: build_error_reason(err), + }); + } + } + } + } + + // check we need another build + self.check_spawn_build().await; + } + + /// check if a build is active + fn is_build_active(&self) -> bool { + self.last_build_started > self.last_build_finished + } + + /// Spawn a new build + async fn spawn_build(&mut self) { + self.last_build_started = Instant::now(); + + let build = self.build.clone(); + let build_tx = self.build_tx.clone(); + + tokio::spawn(async move { + // run the build + let result = build.lock().await.build().await; + // report the result + build_tx.send(result).await + }); + } + + async fn check_spawn_build(&mut self) { + if self.last_change <= self.last_build_started { + tracing::trace!("No changes since last build was started"); + return; + } + + tracing::debug!("Changes since last build was started, checking cooldown"); + + if let Some(cooldown) = self.watcher_cooldown { + let time_since_last_build = self.last_build_finished - self.last_change; + if time_since_last_build < cooldown { + tracing::debug!( + "Cooldown still active: {} remaining", + humantime::Duration::from(cooldown - time_since_last_build) + ); + return; + } + } + + self.spawn_build().await; + } + #[tracing::instrument(level = "trace", skip(self, event))] async fn handle_watch_event(&mut self, event: DebouncedEvent) { - // There are various OS syscalls which can trigger FS changes, even though semantically no - // changes were made. A notorious example which has plagued the trunk watcher - // implementation is `std::fs::copy`, which will trigger watcher changes indicating - // that file contents have been modified. - // - // Given the difficult nature of this issue, we opt for using a cooldown period. Any changes - // events processed within the cooldown period following a build will be ignored. - if Instant::now().duration_since(self.last_build_finished) <= WATCHER_COOLDOWN { - // Purge any other events in the queue. - while let Ok(_event) = self.watch_rx.try_recv() {} + tracing::trace!( + "change detected in {:?} of type {:?}", + event.paths, + event.kind + ); + + if !self.is_event_relevant(&event).await { + tracing::trace!("Event not relevant, skipping"); + return; + } + + // record time of the last accepted change + self.last_change = Instant::now(); + + if self.is_build_active() { + tracing::debug!("Build is active, postponing start"); return; } + // Else, time to trigger a build. + self.check_spawn_build().await; + } + + async fn is_event_relevant(&self, event: &DebouncedEvent) -> bool { // Check each path in the event for a match. match event.event.kind { - EventKind::Modify(ModifyKind::Name(_) | ModifyKind::Data(_)) + EventKind::Modify( + ModifyKind::Name(_) + | ModifyKind::Data(_) + | ModifyKind::Metadata(MetadataKind::WriteTime) + | ModifyKind::Any, + ) | EventKind::Create(_) | EventKind::Remove(_) => (), - _ => return, + _ => return false, }; - let mut found_matching_path = false; + for ev_path in &event.paths { let ev_path = match tokio::fs::canonicalize(&ev_path).await { Ok(ev_path) => ev_path, @@ -150,24 +292,12 @@ impl WatchSystem { } // If all of the above checks have passed, then we need to trigger a build. - tracing::debug!("change detected in {:?} of type {:?}", ev_path, event.kind); - found_matching_path = true; + tracing::debug!("accepted change in {:?} of type {:?}", ev_path, event.kind); + // But we can return early, as we don't need to check the remaining changes + return true; } - // If a build is not needed, then return. - if !found_matching_path { - return; - } - - // Else, time to trigger a build. - let _res = self.build.build().await; - self.last_build_finished = tokio::time::Instant::now(); - - // TODO/NOTE: in the future, we will want to be able to pass along error info and other - // diagnostics info over the socket for use in an error overlay or console logging. - if let Some(tx) = self.build_done_tx.as_mut() { - let _ = tx.send(()); - } + false } fn update_ignore_list(&mut self, arg_path: PathBuf) { @@ -182,13 +312,11 @@ impl WatchSystem { } } -/// Build a FS watcher, when the watcher is dropped, it will stop watching for events. -fn build_watcher( +fn new_debouncer( watch_tx: mpsc::Sender, - paths: Vec, -) -> Result { - // Build the filesystem watcher & debouncer. - let mut debouncer = new_debouncer( + config: Option, +) -> Result> { + new_debouncer_opt::<_, T, FileIdMap>( DEBOUNCE_DURATION, None, move |result: DebounceEventResult| match result { @@ -199,8 +327,34 @@ fn build_watcher( .into_iter() .for_each(|err| tracing::warn!(error=?err, "error from filesystem watcher")), }, + FileIdMap::new(), + config.unwrap_or_default(), ) - .context("failed to build file system watcher")?; + .context("failed to build file system watcher") +} + +/// Build a FS watcher, when the watcher is dropped, it will stop watching for events. +fn build_watcher( + watch_tx: mpsc::Sender, + paths: Vec, + poll: Option, +) -> Result { + // Build the filesystem watcher & debouncer. + + if let Some(duration) = poll { + tracing::info!( + "Running in polling mode: {}", + humantime::Duration::from(duration) + ); + } + + let mut debouncer = match poll { + None => FsDebouncer::Default(new_debouncer::(watch_tx, None)?), + Some(duration) => FsDebouncer::Polling(new_debouncer::( + watch_tx, + Some(notify::Config::default().with_poll_interval(duration)), + )?), + }; // Create a recursive watcher on each of the given paths. // NOTE WELL: it is expected that all given paths are canonical. The Trunk config @@ -214,7 +368,27 @@ fn build_watcher( "failed to watch {:?} for file system changes", path ))?; + debouncer.cache().add_root(&path, RecursiveMode::Recursive); } Ok(debouncer) } + +fn build_error_reason(error: anyhow::Error) -> String { + let mut result = error.to_string(); + result.push_str("\n\n"); + + let mut i = 0usize; + let mut next = error.source(); + while let Some(current) = next { + if i == 0 { + writeln!(&mut result, "Caused by:").unwrap(); + } + writeln!(&mut result, "\t{i}: {current}").unwrap(); + + i += 1; + next = current.source(); + } + + result +} diff --git a/src/ws.rs b/src/ws.rs new file mode 100644 index 00000000..57fcc363 --- /dev/null +++ b/src/ws.rs @@ -0,0 +1,94 @@ +use crate::serve; +use axum::extract::ws::{Message, WebSocket}; +use futures_util::StreamExt; +use std::sync::Arc; +use tokio_stream::wrappers::WatchStream; + +/// (outgoing) communication messages with the websocket +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "camelCase")] +#[serde(tag = "type", content = "data")] +pub enum ClientMessage { + Reload, + BuildFailure { reason: String }, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)] +pub enum State { + #[default] + Ok, + Failed { + reason: String, + }, +} + +pub(crate) async fn handle_ws(mut ws: WebSocket, state: Arc) { + let mut rx = WatchStream::new(state.ws_state.clone()); + tracing::debug!("autoreload websocket opened"); + + let mut first = true; + + loop { + tokio::select! { + msg = ws.recv() => { + match msg { + Some(Ok(Message::Close(reason))) => { + tracing::debug!("received close from browser: {reason:?}"); + let _ = ws.send(Message::Close(reason)).await; + let _ = ws.close().await; + return + } + Some(Ok(msg)) => { + tracing::debug!("received message from browser: {msg:?} (ignoring)"); + } + Some(Err(err))=> { + tracing::debug!("autoreload websocket closed: {err}"); + return + } + None => { + tracing::debug!("lost websocket"); + return + } + } + } + state = rx.next() => { + + let state = match state { + Some(state) => state, + None => { + tracing::debug!("state watcher closed"); + return + }, + }; + + tracing::trace!("Build state changed: {state:?}"); + + let msg = match state { + State::Ok if first => { + // If the state is ok, and it's the first message we would send, discard it, + // as this would cause a reload right after connecting. On the other side, + // we want to send out a failed build even after reconnecting. + first = false; + tracing::trace!("Discarding first reload trigger"); + None + }, + State::Ok => Some(ClientMessage::Reload), + State::Failed { reason } => Some(ClientMessage::BuildFailure { reason }), + }; + + tracing::trace!("Message to send: {msg:?}"); + + if let Some(msg) = msg { + if let Ok(text) = serde_json::to_string(&msg) { + if let Err(err) = ws.send(Message::Text(text)).await { + tracing::info!("autoload websocket failed to send: {err}"); + break; + } + } + } + } + } + } + + tracing::debug!("exiting WS handler"); +}