From e0f8ab5a4eaa0ebf07f68304f6a31b434ec7bea5 Mon Sep 17 00:00:00 2001 From: sinskiy Date: Tue, 24 Dec 2024 20:48:58 +0300 Subject: [PATCH 1/5] fix: add noindex to server island headers --- packages/astro/src/core/render-context.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/astro/src/core/render-context.ts b/packages/astro/src/core/render-context.ts index e001461593e2..ea5ef3ab418b 100644 --- a/packages/astro/src/core/render-context.ts +++ b/packages/astro/src/core/render-context.ts @@ -225,6 +225,10 @@ export class RenderContext { if (this.isRewriting) { response.headers.set(REWRITE_DIRECTIVE_HEADER_KEY, REWRITE_DIRECTIVE_HEADER_VALUE); } + // Prevent server islands from being indexed + if (this.routeData.component === SERVER_ISLAND_COMPONENT) { + response.headers.set('X-Robots-Tag', 'noindex'); + } break; } case 'fallback': { From 36495230fb526c820271bab7e24ab3adf7bc0074 Mon Sep 17 00:00:00 2001 From: sinskiy Date: Tue, 24 Dec 2024 20:59:39 +0300 Subject: [PATCH 2/5] test: check if server island has noindex header --- packages/astro/test/server-islands.test.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/astro/test/server-islands.test.js b/packages/astro/test/server-islands.test.js index 913650a95029..6c55a793c1c2 100644 --- a/packages/astro/test/server-islands.test.js +++ b/packages/astro/test/server-islands.test.js @@ -37,6 +37,11 @@ describe('Server islands', () => { assert.equal(serverIslandEl.length, 0); }); + it('island is not indexed', async () => { + const res = await fixture.fetch('/_server-islands/Island'); + assert.equal(res.headers.get('X-Robots-Tag'), 'noindex'); + }); + it('island can set headers', async () => { const res = await fixture.fetch('/_server-islands/Island', { method: 'POST', @@ -74,6 +79,13 @@ describe('Server islands', () => { const serverIslandScript = $('script[data-island-id]'); assert.equal(serverIslandScript.length, 1, 'has the island script'); }); + + it('island is not indexed', async () => { + const app = await fixture.loadTestAdapterApp(); + const request = new Request('http://example.com/_server-islands/Island'); + const response = await app.render(request); + assert.equal(response.headers.get('X-Robots-Tag'), 'noindex'); + }); }); }); From 2b63e47ab91cd4739ece4dc00c44d0f4dceed0b9 Mon Sep 17 00:00:00 2001 From: sinskiy Date: Tue, 24 Dec 2024 21:13:37 +0300 Subject: [PATCH 3/5] chore: changeset --- .changeset/witty-lies-end.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/witty-lies-end.md diff --git a/.changeset/witty-lies-end.md b/.changeset/witty-lies-end.md new file mode 100644 index 000000000000..d12046eaec1e --- /dev/null +++ b/.changeset/witty-lies-end.md @@ -0,0 +1,5 @@ +--- +'astro': patch +--- + +Fixes an issue when crawlers try to index Server Islands thinking that Server Islands are pages From a743ff2f5380e9313336ee34fd8ff75ca44db87f Mon Sep 17 00:00:00 2001 From: sinskiy Date: Sat, 28 Dec 2024 12:45:18 +0300 Subject: [PATCH 4/5] refactor: set X-Robots-Tag: noindex in handler --- packages/astro/src/core/render-context.ts | 4 ---- packages/astro/src/core/server-islands/endpoint.ts | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/packages/astro/src/core/render-context.ts b/packages/astro/src/core/render-context.ts index ea5ef3ab418b..e001461593e2 100644 --- a/packages/astro/src/core/render-context.ts +++ b/packages/astro/src/core/render-context.ts @@ -225,10 +225,6 @@ export class RenderContext { if (this.isRewriting) { response.headers.set(REWRITE_DIRECTIVE_HEADER_KEY, REWRITE_DIRECTIVE_HEADER_VALUE); } - // Prevent server islands from being indexed - if (this.routeData.component === SERVER_ISLAND_COMPONENT) { - response.headers.set('X-Robots-Tag', 'noindex'); - } break; } case 'fallback': { diff --git a/packages/astro/src/core/server-islands/endpoint.ts b/packages/astro/src/core/server-islands/endpoint.ts index 5afdde651b7d..ad72969fae62 100644 --- a/packages/astro/src/core/server-islands/endpoint.ts +++ b/packages/astro/src/core/server-islands/endpoint.ts @@ -7,6 +7,7 @@ import { import { isAstroComponentFactory } from '../../runtime/server/render/astro/factory.js'; import { createSlotValueFromString } from '../../runtime/server/render/slot.js'; import type { ComponentInstance, ManifestData } from '../../types/astro.js'; +import type { AstroGlobal } from '../../types/public/context.js'; import type { RouteData, SSRManifest } from '../../types/public/internal.js'; import { decryptString } from '../encryption.js'; import { getPattern } from '../routing/manifest/pattern.js'; @@ -104,6 +105,7 @@ export function createEndpoint(manifest: SSRManifest) { // Get the request data from the body or search params const data = await getRequestData(result.request); + // probably error if (data instanceof Response) { return data; } @@ -130,6 +132,9 @@ export function createEndpoint(manifest: SSRManifest) { slots[prop] = createSlotValueFromString(data.slots[prop]); } + // Prevent server islands from being indexed + result.response.headers.set('X-Robots-Tag', 'noindex') + // Wrap Astro components so we can set propagation to // `self` which is needed to force the runtime to wait // on the component before sending out the response headers. From 7667cc2e67d5455015d21aef73e5302cb8804aa8 Mon Sep 17 00:00:00 2001 From: Emanuele Stoppa Date: Thu, 2 Jan 2025 14:28:41 +0000 Subject: [PATCH 5/5] fix tests --- .../astro/src/core/server-islands/endpoint.ts | 3 +-- packages/astro/test/server-islands.test.js | 25 ++++++++++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/packages/astro/src/core/server-islands/endpoint.ts b/packages/astro/src/core/server-islands/endpoint.ts index ad72969fae62..97ea4ee23c9a 100644 --- a/packages/astro/src/core/server-islands/endpoint.ts +++ b/packages/astro/src/core/server-islands/endpoint.ts @@ -7,7 +7,6 @@ import { import { isAstroComponentFactory } from '../../runtime/server/render/astro/factory.js'; import { createSlotValueFromString } from '../../runtime/server/render/slot.js'; import type { ComponentInstance, ManifestData } from '../../types/astro.js'; -import type { AstroGlobal } from '../../types/public/context.js'; import type { RouteData, SSRManifest } from '../../types/public/internal.js'; import { decryptString } from '../encryption.js'; import { getPattern } from '../routing/manifest/pattern.js'; @@ -133,7 +132,7 @@ export function createEndpoint(manifest: SSRManifest) { } // Prevent server islands from being indexed - result.response.headers.set('X-Robots-Tag', 'noindex') + result.response.headers.set('X-Robots-Tag', 'noindex'); // Wrap Astro components so we can set propagation to // `self` which is needed to force the runtime to wait diff --git a/packages/astro/test/server-islands.test.js b/packages/astro/test/server-islands.test.js index 6c55a793c1c2..e0a591588f63 100644 --- a/packages/astro/test/server-islands.test.js +++ b/packages/astro/test/server-islands.test.js @@ -38,8 +38,15 @@ describe('Server islands', () => { }); it('island is not indexed', async () => { - const res = await fixture.fetch('/_server-islands/Island'); - assert.equal(res.headers.get('X-Robots-Tag'), 'noindex'); + const res = await fixture.fetch('/_server-islands/Island', { + method: 'POST', + body: JSON.stringify({ + componentExport: 'default', + encryptedProps: 'FC8337AF072BE5B1641501E1r8mLIhmIME1AV7UO9XmW9OLD', + slots: {}, + }), + }); + assert.equal(res.headers.get('x-robots-tag'), 'noindex'); }); it('island can set headers', async () => { @@ -82,9 +89,19 @@ describe('Server islands', () => { it('island is not indexed', async () => { const app = await fixture.loadTestAdapterApp(); - const request = new Request('http://example.com/_server-islands/Island'); + const request = new Request('http://example.com/_server-islands/Island', { + method: 'POST', + body: JSON.stringify({ + componentExport: 'default', + encryptedProps: 'FC8337AF072BE5B1641501E1r8mLIhmIME1AV7UO9XmW9OLD', + slots: {}, + }), + headers: { + origin: 'http://example.com', + }, + }); const response = await app.render(request); - assert.equal(response.headers.get('X-Robots-Tag'), 'noindex'); + assert.equal(response.headers.get('x-robots-tag'), 'noindex'); }); }); });