Skip to content

Commit

Permalink
Add X-Notion-Space-Id header to resolve HTTP 504 errors (#312)
Browse files Browse the repository at this point in the history
  • Loading branch information
@karlhorky
karlhorky authored Feb 25, 2025
1 parent f28d078 commit d07102c
Show file tree
Hide file tree
Showing 3 changed files with 1,903 additions and 2,882 deletions.
293 changes: 154 additions & 139 deletions index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,176 +54,191 @@ type Task = {
};
};

const client = axios.create({
// Notion unofficial API
baseURL: 'https://www.notion.so/api/v3',
headers: {
Cookie: `token_v2=${process.env.NOTION_TOKEN}`,
},
});
for (const [spaceId, spaceBlocks] of Object.entries(
Object.groupBy(blocks, (block) => block.spaceId),
)) {
if (!spaceBlocks) continue;

function delay(ms: number) {
console.log(
`Waiting ${ms / 1000} second${ms > 1000 ? 's' : ''} before polling again...`,
`Exporting ${spaceBlocks.length} blocks from space ${spaceId}...`,
);
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}

// Enqueue all export tasks immediately, without waiting for the
// export tasks to complete
const enqueuedBlocks = await pMap(blocks, async (block) => {
const {
data: { taskId },
}: { data: { taskId: string } } = await client.post('enqueueTask', {
task: {
eventName: 'exportBlock',
request: {
block: {
id: block.id,
spaceId: block.spaceId,
},
exportOptions: {
exportType: 'markdown',
locale: 'en',
timeZone: 'Europe/Vienna',
},
recursive: block.recursive,
},
const client = axios.create({
// Notion unofficial API
baseURL: 'https://www.notion.so/api/v3',
headers: {
Cookie: `token_v2=${process.env.NOTION_TOKEN}`,
'X-Notion-Space-Id': spaceId,
},
});

if (!taskId) {
throw new Error('No taskId returned from enqueueTask');
function delay(ms: number) {
console.log(
`Waiting ${ms / 1000} second${ms > 1000 ? 's' : ''} before polling again...`,
);
return new Promise((resolve) => {
setTimeout(resolve, ms);
});
}

console.log(`Started export of block ${block.dirName} as task ${taskId}`);
// Enqueue all export tasks immediately, without waiting for the
// export tasks to complete
const enqueuedBlocks = await pMap(spaceBlocks, async (block) => {
const {
data: { taskId },
}: { data: { taskId: string } } = await client.post('enqueueTask', {
task: {
eventName: 'exportBlock',
request: {
block: {
id: block.id,
spaceId: block.spaceId,
},
exportOptions: {
exportType: 'markdown',
locale: 'en',
timeZone: 'Europe/Vienna',
},
recursive: block.recursive,
},
},
});

const task: BlockTask = {
id: taskId,
state: null,
status: {
pagesExported: null,
exportURL: null,
},
};
if (!taskId) {
throw new Error('No taskId returned from enqueueTask');
}

return {
...block,
task: task,
};
});
console.log(`Started export of block ${block.dirName} as task ${taskId}`);

let retries = 0;
const task: BlockTask = {
id: taskId,
state: null,
status: {
pagesExported: null,
exportURL: null,
},
};

// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
const incompleteEnqueuedBlocks = enqueuedBlocks.filter(
({ task }) => task.state !== 'success',
);
return {
...block,
task: task,
};
});

const taskIds = incompleteEnqueuedBlocks.map(({ task }) => task.id);
let retries = 0;

try {
const {
data: { results },
headers: { 'set-cookie': getTasksRequestCookies },
}: { data: { results: Task[] }; headers: { 'set-cookie': string[] } } =
await client.post('getTasks', {
taskIds: taskIds,
});

const blocksWithTaskProgress = results.reduce(
(blocksAcc, task) => {
const block = enqueuedBlocks.find(({ task: { id } }) => id === task.id);

if (!block || !task.status) return blocksAcc;

// Mutate original object in enqueuedBlocks for while loop
// exit condition
block.task.state = task.state;
block.task.status.pagesExported = task.status.pagesExported;
block.task.status.exportURL = task.status.exportURL;

return blocksAcc.concat(block);
},
[] as typeof incompleteEnqueuedBlocks,
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
while (true) {
const incompleteEnqueuedBlocks = enqueuedBlocks.filter(
({ task }) => task.state !== 'success',
);

for (const block of blocksWithTaskProgress) {
console.log(
`Exported ${block.task.status.pagesExported} pages for ${block.dirName}`,
);
const taskIds = incompleteEnqueuedBlocks.map(({ task }) => task.id);

if (block.task.state === 'success') {
const backupDirPath = path.join(
process.cwd(),
'exports',
block.dirName,
);
try {
const {
data: { results },
headers: { 'set-cookie': getTasksRequestCookies },
}: { data: { results: Task[] }; headers: { 'set-cookie': string[] } } =
await client.post('getTasks', {
taskIds: taskIds,
});

const temporaryZipPath = path.join(
process.cwd(),
'exports',
`${block.dirName}.zip`,
);
const blocksWithTaskProgress = results.reduce(
(blocksAcc, task) => {
const block = enqueuedBlocks.find(
({ task: { id } }) => id === task.id,
);

console.log(`Export finished for ${block.dirName}`);
if (!block || !task.status) return blocksAcc;

const response = await client<Stream>({
method: 'GET',
url: block.task.status.exportURL || undefined,
responseType: 'stream',
headers: {
Cookie: getTasksRequestCookies.find((cookie) =>
cookie.includes('file_token='),
),
},
});

const sizeInMb =
Number(response.headers['content-length']) / 1000 / 1000;
console.log(`Downloading ${Math.round(sizeInMb * 1000) / 1000}mb...`);
// Mutate original object in enqueuedBlocks for while loop
// exit condition
block.task.state = task.state;
block.task.status.pagesExported = task.status.pagesExported;
block.task.status.exportURL = task.status.exportURL;

const stream = response.data.pipe(createWriteStream(temporaryZipPath));
return blocksAcc.concat(block);
},
[] as typeof incompleteEnqueuedBlocks,
);

await new Promise((resolve, reject) => {
stream.on('close', resolve);
stream.on('error', reject);
});
for (const block of blocksWithTaskProgress) {
console.log(
`Exported ${block.task.status.pagesExported} pages for ${block.dirName}`,
);

rmSync(backupDirPath, { recursive: true, force: true });
mkdirSync(backupDirPath, { recursive: true });
await extract(temporaryZipPath, { dir: backupDirPath });
unlinkSync(temporaryZipPath);
if (block.task.state === 'success') {
const backupDirPath = path.join(
process.cwd(),
'exports',
block.dirName,
);

const temporaryZipPath = path.join(
process.cwd(),
'exports',
`${block.dirName}.zip`,
);

console.log(`Export finished for ${block.dirName}`);

const response = await client<Stream>({
method: 'GET',
url: block.task.status.exportURL || undefined,
responseType: 'stream',
headers: {
Cookie: getTasksRequestCookies.find((cookie) =>
cookie.includes('file_token='),
),
},
});

const sizeInMb =
Number(response.headers['content-length']) / 1000 / 1000;
console.log(`Downloading ${Math.round(sizeInMb * 1000) / 1000}mb...`);

const stream = response.data.pipe(
createWriteStream(temporaryZipPath),
);

await new Promise((resolve, reject) => {
stream.on('close', resolve);
stream.on('error', reject);
});

rmSync(backupDirPath, { recursive: true, force: true });
mkdirSync(backupDirPath, { recursive: true });
await extract(temporaryZipPath, { dir: backupDirPath });
unlinkSync(temporaryZipPath);

console.log(`✅ Export of ${block.dirName} downloaded and unzipped`);
}
}

console.log(`✅ Export of ${block.dirName} downloaded and unzipped`);
// If all blocks are done, break out of the loop
if (!enqueuedBlocks.find(({ task }) => task.state !== 'success')) {
break;
}
}

// If all blocks are done, break out of the loop
if (!enqueuedBlocks.find(({ task }) => task.state !== 'success')) {
break;
}
// Reset retries on success
retries = 0;
} catch (error) {
if (!axios.isAxiosError(error) || error.response?.status !== 429) {
// Rethrow errors which do not contain an HTTP 429 status
// code
throw error;
}

// Reset retries on success
retries = 0;
} catch (error) {
if (!axios.isAxiosError(error) || error.response?.status !== 429) {
// Rethrow errors which do not contain an HTTP 429 status
// code
throw error;
console.log(
'Received response with HTTP 429 (Too Many Requests), increasing delay...',
);
retries += 1;
}

console.log(
'Received response with HTTP 429 (Too Many Requests), increasing delay...',
);
retries += 1;
// Rate limit polling, with incremental backoff
await delay(1000 + 1000 * retries);
}

// Rate limit polling, with incremental backoff
await delay(1000 + 1000 * retries);
}

console.log('✅ All exports successful');
2 changes: 1 addition & 1 deletion package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
},
"devDependencies": {
"eslint": "9.20.1",
"eslint-config-upleveled": "9.1.2",
"eslint-config-upleveled": "9.2.0",
"prettier": "3.5.1",
"typescript": "5.7.3"
},
Expand Down
Loading

0 comments on commit d07102c

Please sign in to comment.