ci: push tiles and styles into date folder in R2

.github/workflows/tiles-file.yml

@@ -9,21 +9,41 @@ on:
     branches: [ "main" ]
     paths:
       - ".github/workflows/tiles-file.yml"
+      - "styles-gen/**"
 
 concurrency:
   group: ${{ github.workflow }}
   cancel-in-progress: true
 
 jobs:
   deploy_tiles:
-    name: Deploy Tiles
+    name: Deploy Tiles & Styles
     strategy:
       matrix:
         bucket_region: [ "wnam", "enam", "weur", "eeur", "apac" ]
+    defaults:
+      run:
+        working-directory: 'styles-gen'
     runs-on: ubuntu-latest
     outputs:
       KEY: ${{ steps.latest_build.outputs.KEY }}
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version-file: '.nvmrc'
+
+      - name: Install dependencies
+        run: npm ci
+
+      - name: Generate styles
+        run: npm run generate-styles
+
       - name: Find latest pmtiles build
         id: latest_build
         env:
@@ -34,6 +54,7 @@ jobs:
           else
             build=$FILE_NAME
           fi
+          export build=${build%.pmtiles}
           echo "KEY=$build"
           echo "KEY=$build" >> $GITHUB_OUTPUT
 
@@ -53,8 +74,11 @@ jobs:
             type = http
             url = https://build.protomaps.com/
 
+      - name: Copy latest styles to R2
+        run: rclone --progress copy ./dist/ r2:/tiles-${{ matrix.bucket_region }}/${{ steps.latest_build.outputs.KEY }}/styles/ --s3-no-check-bucket --s3-chunk-size=64M --multi-thread-streams=50 --checkers=512
+
       - name: Copy latest tiles to R2
-        run: rclone --progress copyto pmtiles:/${{ steps.latest_build.outputs.KEY }} r2:/tiles-${{ matrix.bucket_region }}/${{ steps.latest_build.outputs.KEY }} --s3-no-check-bucket --s3-chunk-size=64M --multi-thread-streams=50 --checkers=512
+        run: rclone --progress copyto pmtiles:/${{ steps.latest_build.outputs.KEY }}.pmtiles r2:/tiles-${{ matrix.bucket_region }}/${{ steps.latest_build.outputs.KEY }}/tiles.pmtiles --s3-no-check-bucket --s3-chunk-size=64M --multi-thread-streams=50 --checkers=512
   update_tile_ref:
     name: Update tile server file reference
     needs: [deploy_tiles]
@@ -74,7 +98,7 @@ jobs:
 
       - name: Update deployment file name
         run: |
-          sed -i -E 's/"pmtiles_file_name": "(.*)"/"pmtiles_file_name": "${{ needs.deploy_tiles.outputs.KEY }}"/' deployment/modules/cloudflare/tiles-worker/tiles.tfvars.json
+          sed -i -E 's/"pmtiles_deployment_key": "(.*)"/"pmtiles_deployment_key": "${{ needs.deploy_tiles.outputs.KEY }}"/' deployment/modules/cloudflare/tiles-worker/tiles.tfvars.json
 
       - name: Push updated tiles.tfvars to repo
         uses: EndBug/add-and-commit@v9

deployment/modules/cloudflare/tiles-worker/tiles.tfvars.json

@@ -1,3 +1,3 @@
 {
-  "pmtiles_file_name": "20240812.pmtiles"
+  "pmtiles_deployment_key": "20240812"
 }

deployment/modules/cloudflare/tiles-worker/variables.tf

@@ -1,6 +1,6 @@
 variable "cloudflare_account_id" {}
 variable "tf_state_postgres_conn_str" {}
-variable "pmtiles_file_name" {}
+variable "pmtiles_deployment_key" {}
 variable "tiles_build_dir" {}
 variable "vmetrics_api_token" {}
 variable "env" {}

deployment/modules/cloudflare/tiles-worker/workers.tf

@@ -10,7 +10,7 @@ resource "cloudflare_workers_script" "tiles" {
 
   plain_text_binding {
     name = "PMTILES_FILE_NAME"
-    text = var.pmtiles_file_name
+    text = var.pmtiles_deployment_key
   }
 
   plain_text_binding {

tiles/src/env.d.ts

@@ -1,5 +1,5 @@
-interface WorkerEnv extends Omit<Env, 'ENVIRONMENT' | 'PMTILES_FILE_NAME'> {
+interface WorkerEnv extends Omit<Env, 'ENVIRONMENT' | 'DEPLOYMENT_KEY'> {
   ENVIRONMENT: string;
-  PMTILES_FILE_NAME: string;
+  DEPLOYMENT_KEY: string;
   VMETRICS_API_TOKEN: string;
 }

tiles/src/index.ts

@@ -38,7 +38,7 @@ type PMTilesJsonParams = PMTilesParams & {
 };
 
 enum Header {
-  PMTILES_FILE_IDENTIFIER = 'PMTiles-File-Identifier',
+  PMTILES_DEPLOYMENT_KEY = 'PMTiles-Deployment-Key',
   CACHE_CONTROL = 'Cache-Control',
   ACCESS_CONTROL_ALLOW_ORIGIN = 'Access-Control-Allow-Origin',
   VARY = 'Vary',
@@ -102,7 +102,7 @@ async function handleRequest(
   const cached = await metrics.monitorAsyncFunction({ name: 'match_request_from_cdn' }, (url) => cache.match(url))(
     request.url,
   );
-  if (cached && cached.headers.get(Header.PMTILES_FILE_IDENTIFIER) === env.PMTILES_FILE_NAME) {
+  if (cached && cached.headers.get(Header.PMTILES_DEPLOYMENT_KEY) === env.DEPLOYMENT_KEY) {
     const cacheHeaders = new Headers(cached.headers);
     const encodeBody = cacheHeaders.has('content-encoding') ? 'manual' : 'automatic';
     return new Response(cached.body, {
@@ -132,7 +132,7 @@ async function handleRequest(
     Object.entries(bucketMap).filter(([key]) => buckets.includes(key as R2BucketRegion)),
   );
 
-  const storageRepository = new R2StorageRepository(filteredBucketMap, env.PMTILES_FILE_NAME, metrics);
+  const storageRepository = new R2StorageRepository(filteredBucketMap, env.DEPLOYMENT_KEY, metrics);
   const pmTilesService = await metrics.monitorAsyncFunction({ name: 'pmtiles_init' }, PMTilesService.init)(
     storageRepository,
     memCacheRepository,
@@ -144,7 +144,7 @@ async function handleRequest(
   respHeaders.set(Header.CACHE_CONTROL, `public, max-age=${60 * 60 * 24 * 31}`);
   respHeaders.set(Header.ACCESS_CONTROL_ALLOW_ORIGIN, '*');
   respHeaders.set(Header.VARY, 'Origin');
-  respHeaders.set(Header.PMTILES_FILE_IDENTIFIER, env.PMTILES_FILE_NAME);
+  respHeaders.set(Header.PMTILES_DEPLOYMENT_KEY, env.DEPLOYMENT_KEY);
 
   const pmTilesParams = parseUrl(request);
 

tiles/src/repository.ts

@@ -42,20 +42,20 @@ export class MemCacheRepository implements IMemCacheRepository {
 export class R2StorageRepository implements IStorageRepository {
   constructor(
     private buckets: { [key: string]: R2Bucket },
-    private fileName: string,
+    private deploymentKey: string,
     private metrics: IMetricsRepository,
   ) {}
 
   getFileName(): string {
-    return this.fileName;
+    return this.deploymentKey;
   }
 
   private async getR2Object(range: { offset: number; length: number }) {
     const metric = Metric.create('r2_storage_get');
     const { offset, length } = range;
     const { key: bucketKey, resp } = await Promise.race(
       Object.entries(this.buckets).map(async ([key, bucket]) => {
-        const resp = await bucket.get(this.fileName, {
+        const resp = await bucket.get(`${this.deploymentKey}/tiles.pmtiles`, {
           range: { offset, length },
         });
         return { key, resp };

tiles/test/index.spec.ts

@@ -16,7 +16,7 @@ describe('integration tests', () => {
           throw new Error('File body is undefined');
         }
         const body = file.body as ReadableStream;
-        await bucket.put(env.PMTILES_FILE_NAME, body);
+        await bucket.put(`${env.DEPLOYMENT_KEY}/tiles.pmtiles`, body);
         console.log('File uploaded');
       }
     }, 30000);

tiles/worker-configuration.d.ts

@@ -3,7 +3,7 @@
 
 interface Env {
 	KV: KVNamespace;
-	PMTILES_FILE_NAME: "v1.pmtiles";
+	DEPLOYMENT_KEY: "20240812";
 	ENVIRONMENT: "prod";
 	BUCKET_WNAM: R2Bucket;
 	BUCKET_ENAM: R2Bucket;

tiles/wrangler.toml

@@ -40,5 +40,5 @@ preview_bucket_name = "tiles-apac"
 
 # Define environment variables for your Workers script
 [vars]
-PMTILES_FILE_NAME = "v1.pmtiles"
+DEPLOYMENT_KEY = "20240812"
 ENVIRONMENT = "production"