Skip to content

Commit

Permalink
Fix gitbook connector issues
Browse files Browse the repository at this point in the history
  • Loading branch information
Weves committed Feb 20, 2025
1 parent 4958a53 commit 599b770
Showing 1 changed file with 13 additions and 7 deletions.
20 changes: 13 additions & 7 deletions backend/onyx/connectors/gitbook/connector.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,16 +229,20 @@ def _fetch_all_pages(

try:
content = self.client.get(f"/spaces/{self.space_id}/content")
pages = content.get("pages", [])

pages: list[dict[str, Any]] = content.get("pages", [])
current_batch: list[Document] = []
for page in pages:
updated_at = datetime.fromisoformat(page["updatedAt"])

while pages:
page = pages.pop(0)

updated_at_raw = page.get("updatedAt")
if updated_at_raw is None:
# if updatedAt is not present, that means the page has never been edited
continue

updated_at = datetime.fromisoformat(updated_at_raw)
if start and updated_at < start:
if current_batch:
yield current_batch
return
continue
if end and updated_at > end:
continue

Expand All @@ -250,6 +254,8 @@ def _fetch_all_pages(
yield current_batch
current_batch = []

pages.extend(page.get("pages", []))

if current_batch:
yield current_batch

Expand Down

0 comments on commit 599b770

Please sign in to comment.