Merge pull request #4 from ydennisy/chore/simplify-db-schema

chore: refactor db schema to simple flat tables
ydennisy · Jan 7, 2024 · cdca199 · cdca199
2 parents aeec37b + 26076ef
commit cdca199
Show file tree

Hide file tree

Showing 33 changed files with 900 additions and 595 deletions.
diff --git a/backend/package-lock.json b/backend/package-lock.json
diff --git a/backend/package.json b/backend/package.json
@@ -15,6 +15,7 @@
     "test": "tap run"
   },
   "devDependencies": {
+    "@types/jsdom": "^21.1.6",
     "@types/node": "^20.6.3",
     "nodemon": "^3.0.1",
     "prisma": "^5.6.0",
@@ -23,10 +24,12 @@
     "typescript": "^5.2.2"
   },
   "dependencies": {
+    "@mozilla/readability": "^0.4.4",
     "@prisma/client": "^5.6.0",
     "entities": "^4.5.0",
     "fastify": "^4.24.3",
     "got-scraping": "^3.2.15",
+    "jsdom": "^22.1.0",
     "node-html-parser": "^6.1.10",
     "openai": "^4.15.0",
     "pgvector": "^0.1.5"

diff --git a/backend/prisma/migrations/20231119091508_init_tables/migration.sql b/backend/prisma/migrations/20231119091508_init_tables/migration.sql
diff --git a/backend/prisma/migrations/20231119091949_add_initial_counter/migration.sql b/backend/prisma/migrations/20231119091949_add_initial_counter/migration.sql
diff --git a/backend/prisma/migrations/20231126122313_init_tables/migration.sql b/backend/prisma/migrations/20231126122313_init_tables/migration.sql
@@ -0,0 +1,48 @@
+-- CreateTable
+CREATE TABLE "web_pages" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    "url" TEXT NOT NULL,
+    "title" TEXT NOT NULL,
+    "domain" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "embedding" vector(1536),
+
+    CONSTRAINT "web_pages_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "notes" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    "title" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "embedding" vector(1536),
+
+    CONSTRAINT "notes_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "papers" (
+    "id" TEXT NOT NULL,
+    "created_at" TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP,
+    "updated_at" TIMESTAMP(3) NOT NULL,
+    "title" TEXT NOT NULL,
+    "author" TEXT NOT NULL,
+    "url" TEXT NOT NULL,
+    "content" TEXT NOT NULL,
+    "embedding" vector(1536),
+
+    CONSTRAINT "papers_pkey" PRIMARY KEY ("id")
+);
+
+-- CreateTable
+CREATE TABLE "counters" (
+    "id" SERIAL NOT NULL,
+    "name" TEXT NOT NULL,
+    "value" INTEGER NOT NULL,
+
+    CONSTRAINT "counters_pkey" PRIMARY KEY ("id")
+);
diff --git a/backend/prisma/schema.prisma b/backend/prisma/schema.prisma
@@ -8,63 +8,47 @@ datasource db {
   directUrl = env("DATABASE_URL_DIRECT")
 }
 
-model Node {
-  id           Int                          @id @default(autoincrement())
-  createdAt    DateTime                     @default(now()) @map("created_at")
-  updatedAt    DateTime                     @updatedAt @map("updated_at")
-  raw          String
-  title        String?
-  type         NodeType
-  embedding    Unsupported("vector(1536)")?
-  nodeParents  Edge[]                       @relation("child_to_parent")
-  nodeChildren Edge[]                       @relation("parent_to_child")
-  note         Note?
-  webPage      WebPage?
-
-  @@index(id)
-  @@map("nodes")
-}
-
-model Edge {
-  id       Int  @id @default(autoincrement())
-  parent   Node @relation("parent_to_child", fields: [parentId], references: [id])
-  parentId Int  @map("parent_id")
-  child    Node @relation("child_to_parent", fields: [childId], references: [id])
-  childId  Int  @map("child_id")
-
-  @@unique([parentId, childId])
-  @@map("edges")
-}
-
 model WebPage {
-  id     Int    @id @default(autoincrement())
-  url    String
-  title  String
-  node   Node   @relation(fields: [nodeId], references: [id])
-  nodeId Int    @unique @map("node_id")
+  id        String                       @id
+  createdAt DateTime                     @default(now()) @map("created_at")
+  updatedAt DateTime                     @updatedAt @map("updated_at")
+  url       String
+  title     String
+  domain    String
+  content   String
+  embedding Unsupported("vector(1536)")?
 
   @@map("web_pages")
 }
 
 model Note {
-  id     Int    @id @default(autoincrement())
-  title  String
-  body   String
-  node   Node   @relation(fields: [nodeId], references: [id])
-  nodeId Int    @unique @map("node_id")
+  id        String                       @id
+  createdAt DateTime                     @default(now()) @map("created_at")
+  updatedAt DateTime                     @updatedAt @map("updated_at")
+  title     String
+  content   String
+  embedding Unsupported("vector(1536)")?
 
   @@map("notes")
 }
 
+model Paper {
+  id        String                       @id
+  createdAt DateTime                     @default(now()) @map("created_at")
+  updatedAt DateTime                     @updatedAt @map("updated_at")
+  title     String
+  author    String
+  url       String
+  content   String
+  embedding Unsupported("vector(1536)")?
+
+  @@map("papers")
+}
+
 model Counter {
   id    Int    @id @default(autoincrement())
   name  String
   value Int
 
   @@map("counters")
 }
-
-enum NodeType {
-  NOTE
-  WEB_PAGE
-}