updated backend to next.js

backend/pages/api/ingest_docs.js

@@ -0,0 +1,79 @@
+import weaviate from 'weaviate-client';
+import fs from 'fs';
+import path from 'path';
+import { JSDOM } from 'jsdom';
+import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter';
+import { Weaviate as WeaviateStore } from 'langchain_community/vectorstores';
+import { OpenAIEmbeddings } from 'langchain_openai';
+import { SQLRecordManager, index } from 'langchain/indexes';
+
+export default async function handler(req, res) {
+  try {
+    const WEAVIATE_URL = process.env.WEAVIATE_URL;
+    const WEAVIATE_API_KEY = process.env.WEAVIATE_API_KEY;
+    const RECORD_MANAGER_DB_URL = process.env.RECORD_MANAGER_DB_URL;
+    const MY_HTML_FILES_DIRECTORY = process.env.MY_HTML_FILES_DIRECTORY;
+    const WEAVIATE_DOCS_INDEX_NAME = process.env.WEAVIATE_DOCS_INDEX_NAME;
+
+    const textSplitter = new RecursiveCharacterTextSplitter({ chunk_size: 4000, chunk_overlap: 200 });
+    const embedding = new OpenAIEmbeddings({ model: "text-embedding-ada-002", chunk_size: 200 });
+
+    const client = weaviate.client({
+      scheme: 'https',
+      host: WEAVIATE_URL,
+      apiKey: new weaviate.ApiKey(WEAVIATE_API_KEY),
+    });
+
+    const vectorstore = new WeaviateStore({
+      client,
+      indexName: WEAVIATE_DOCS_INDEX_NAME,
+      textKey: 'text',
+      embedding,
+      byText: false,
+      attributes: ['source', 'title'],
+    });
+
+    const recordManager = new SQLRecordManager({
+      db_url: RECORD_MANAGER_DB_URL,
+      tableName: `weaviate/${WEAVIATE_DOCS_INDEX_NAME}`,
+    });
+    await recordManager.createSchema();
+
+    const documents = [];
+    const files = fs.readdirSync(MY_HTML_FILES_DIRECTORY);
+
+    for (const file of files) {
+      if (file.endsWith('.html')) {
+        const filePath = path.join(MY_HTML_FILES_DIRECTORY, file);
+        const htmlContent = fs.readFileSync(filePath, 'utf-8');
+        const dom = new JSDOM(htmlContent);
+        const text = dom.window.document.body.textContent.trim();
+        const metadata = {
+          source: filePath,
+          title: dom.window.document.title,
+          description: dom.window.document.querySelector('meta[name="description"]')?.content || '',
+          language: dom.window.document.documentElement.lang || '',
+        };
+        documents.push({ page_content: text, metadata });
+      }
+    }
+
+    const docsTransformed = textSplitter.splitDocuments(documents).filter(doc => doc.page_content.length > 10);
+
+    for (const doc of docsTransformed) {
+      if (!doc.metadata.source) doc.metadata.source = '';
+      if (!doc.metadata.title) doc.metadata.title = '';
+    }
+
+    const indexingStats = await index(docsTransformed, recordManager, vectorstore, {
+      cleanup: 'full',
+      source_id_key: 'source',
+      force_update: (process.env.FORCE_UPDATE || 'false').toLowerCase() === 'true',
+    });
+
+    res.status(200).json({ message: 'Ingestion successful', stats: indexingStats });
+  } catch (error) {
+    console.error(error);
+    res.status(500).json({ error: error.message });
+  }
+}

backend/pages/api/package.json

@@ -0,0 +1,23 @@
+{
+    "name": "chat-langchain",
+    "version": "1.0.0",
+    "description": "Backend serverless function for chat-langchain project",
+    "scripts": {
+      "dev": "next dev",
+      "build": "next build",
+      "start": "next start"
+    },
+    "dependencies": {
+      "next": "latest",
+      "react": "latest",
+      "react-dom": "latest",
+      "weaviate-client": "^3.5.0",
+      "fs": "^0.0.1-security",
+      "path": "^0.12.7",
+      "jsdom": "^21.1.0",
+      "langchain": "^0.1.0",
+      "langchain_community": "^0.1.0",
+      "langchain_openai": "^0.1.0"
+    }
+  }
+

backend/pages/api/vercel.json

@@ -0,0 +1,27 @@
+{
+    "version": 2,
+    "builds": [
+      {
+        "src": "next.config.js",
+        "use": "@vercel/next"
+      },
+      {
+        "src": "pages/api/**/*.js",
+        "use": "@vercel/node"
+      }
+    ],
+    "routes": [
+      {
+        "src": "/api/(.*)",
+        "dest": "/api/$1"
+      }
+    ],
+    "env": {
+      "WEAVIATE_URL": "@weaviate_url",
+      "WEAVIATE_API_KEY": "@weaviate_api_key",
+      "RECORD_MANAGER_DB_URL": "@record_manager_db_url",
+      "MY_HTML_FILES_DIRECTORY": "@my_html_files_directory",
+      "WEAVIATE_DOCS_INDEX_NAME": "@weaviate_docs_index_name"
+    }
+  }
+

nextjs/.eslintrc.json

@@ -0,0 +1,3 @@
+{
+  "extends": "next/core-web-vitals"
+}

nextjs/.gitignore

@@ -0,0 +1,36 @@
+# See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
+
+# dependencies
+/node_modules
+/.pnp
+.pnp.js
+.yarn/install-state.gz
+
+# testing
+/coverage
+
+# next.js
+/.next/
+/out/
+
+# production
+/build
+
+# misc
+.DS_Store
+*.pem
+
+# debug
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+
+# local env files
+.env*.local
+
+# vercel
+.vercel
+
+# typescript
+*.tsbuildinfo
+next-env.d.ts

nextjs/README.md

@@ -0,0 +1,36 @@
+This is a [Next.js](https://nextjs.org/) project bootstrapped with [`create-next-app`](https://github.com/vercel/next.js/tree/canary/packages/create-next-app).
+
+## Getting Started
+
+First, run the development server:
+
+```bash
+npm run dev
+# or
+yarn dev
+# or
+pnpm dev
+# or
+bun dev
+```
+
+Open [http://localhost:3000](http://localhost:3000) with your browser to see the result.
+
+You can start editing the page by modifying `app/page.tsx`. The page auto-updates as you edit the file.
+
+This project uses [`next/font`](https://nextjs.org/docs/basic-features/font-optimization) to automatically optimize and load Inter, a custom Google Font.
+
+## Learn More
+
+To learn more about Next.js, take a look at the following resources:
+
+- [Next.js Documentation](https://nextjs.org/docs) - learn about Next.js features and API.
+- [Learn Next.js](https://nextjs.org/learn) - an interactive Next.js tutorial.
+
+You can check out [the Next.js GitHub repository](https://github.com/vercel/next.js/) - your feedback and contributions are welcome!
+
+## Deploy on Vercel
+
+The easiest way to deploy your Next.js app is to use the [Vercel Platform](https://vercel.com/new?utm_medium=default-template&filter=next.js&utm_source=create-next-app&utm_campaign=create-next-app-readme) from the creators of Next.js.
+
+Check out our [Next.js deployment documentation](https://nextjs.org/docs/deployment) for more details.

nextjs/app/globals.css

@@ -0,0 +1,33 @@
+@tailwind base;
+@tailwind components;
+@tailwind utilities;
+
+:root {
+  --foreground-rgb: 0, 0, 0;
+  --background-start-rgb: 214, 219, 220;
+  --background-end-rgb: 255, 255, 255;
+}
+
+@media (prefers-color-scheme: dark) {
+  :root {
+    --foreground-rgb: 255, 255, 255;
+    --background-start-rgb: 0, 0, 0;
+    --background-end-rgb: 0, 0, 0;
+  }
+}
+
+body {
+  color: rgb(var(--foreground-rgb));
+  background: linear-gradient(
+      to bottom,
+      transparent,
+      rgb(var(--background-end-rgb))
+    )
+    rgb(var(--background-start-rgb));
+}
+
+@layer utilities {
+  .text-balance {
+    text-wrap: balance;
+  }
+}

nextjs/app/layout.tsx

@@ -0,0 +1,22 @@
+import type { Metadata } from "next";
+import { Inter } from "next/font/google";
+import "./globals.css";
+
+const inter = Inter({ subsets: ["latin"] });
+
+export const metadata: Metadata = {
+  title: "Create Next App",
+  description: "Generated by create next app",
+};
+
+export default function RootLayout({
+  children,
+}: Readonly<{
+  children: React.ReactNode;
+}>) {
+  return (
+    <html lang="en">
+      <body className={inter.className}>{children}</body>
+    </html>
+  );
+}

nextjs/app/page.tsx

@@ -0,0 +1,113 @@
+import Image from "next/image";
+
+export default function Home() {
+  return (
+    <main className="flex min-h-screen flex-col items-center justify-between p-24">
+      <div className="z-10 w-full max-w-5xl items-center justify-between font-mono text-sm lg:flex">
+        <p className="fixed left-0 top-0 flex w-full justify-center border-b border-gray-300 bg-gradient-to-b from-zinc-200 pb-6 pt-8 backdrop-blur-2xl dark:border-neutral-800 dark:bg-zinc-800/30 dark:from-inherit lg:static lg:w-auto  lg:rounded-xl lg:border lg:bg-gray-200 lg:p-4 lg:dark:bg-zinc-800/30">
+          Get started by editing&nbsp;
+          <code className="font-mono font-bold">app/page.tsx</code>
+        </p>
+        <div className="fixed bottom-0 left-0 flex h-48 w-full items-end justify-center bg-gradient-to-t from-white via-white dark:from-black dark:via-black lg:static lg:size-auto lg:bg-none">
+          <a
+            className="pointer-events-none flex place-items-center gap-2 p-8 lg:pointer-events-auto lg:p-0"
+            href="https://vercel.com?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
+            target="_blank"
+            rel="noopener noreferrer"
+          >
+            By{" "}
+            <Image
+              src="/vercel.svg"
+              alt="Vercel Logo"
+              className="dark:invert"
+              width={100}
+              height={24}
+              priority
+            />
+          </a>
+        </div>
+      </div>
+
+      <div className="relative z-[-1] flex place-items-center before:absolute before:h-[300px] before:w-full before:-translate-x-1/2 before:rounded-full before:bg-gradient-radial before:from-white before:to-transparent before:blur-2xl before:content-[''] after:absolute after:-z-20 after:h-[180px] after:w-full after:translate-x-1/3 after:bg-gradient-conic after:from-sky-200 after:via-blue-200 after:blur-2xl after:content-[''] before:dark:bg-gradient-to-br before:dark:from-transparent before:dark:to-blue-700 before:dark:opacity-10 after:dark:from-sky-900 after:dark:via-[#0141ff] after:dark:opacity-40 sm:before:w-[480px] sm:after:w-[240px] before:lg:h-[360px]">
+        <Image
+          className="relative dark:drop-shadow-[0_0_0.3rem_#ffffff70] dark:invert"
+          src="/next.svg"
+          alt="Next.js Logo"
+          width={180}
+          height={37}
+          priority
+        />
+      </div>
+
+      <div className="mb-32 grid text-center lg:mb-0 lg:w-full lg:max-w-5xl lg:grid-cols-4 lg:text-left">
+        <a
+          href="https://nextjs.org/docs?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
+          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          <h2 className="mb-3 text-2xl font-semibold">
+            Docs{" "}
+            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
+              -&gt;
+            </span>
+          </h2>
+          <p className="m-0 max-w-[30ch] text-sm opacity-50">
+            Find in-depth information about Next.js features and API.
+          </p>
+        </a>
+
+        <a
+          href="https://nextjs.org/learn?utm_source=create-next-app&utm_medium=appdir-template-tw&utm_campaign=create-next-app"
+          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          <h2 className="mb-3 text-2xl font-semibold">
+            Learn{" "}
+            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
+              -&gt;
+            </span>
+          </h2>
+          <p className="m-0 max-w-[30ch] text-sm opacity-50">
+            Learn about Next.js in an interactive course with&nbsp;quizzes!
+          </p>
+        </a>
+
+        <a
+          href="https://vercel.com/templates?framework=next.js&utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
+          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          <h2 className="mb-3 text-2xl font-semibold">
+            Templates{" "}
+            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
+              -&gt;
+            </span>
+          </h2>
+          <p className="m-0 max-w-[30ch] text-sm opacity-50">
+            Explore starter templates for Next.js.
+          </p>
+        </a>
+
+        <a
+          href="https://vercel.com/new?utm_source=create-next-app&utm_medium=appdir-template&utm_campaign=create-next-app"
+          className="group rounded-lg border border-transparent px-5 py-4 transition-colors hover:border-gray-300 hover:bg-gray-100 hover:dark:border-neutral-700 hover:dark:bg-neutral-800/30"
+          target="_blank"
+          rel="noopener noreferrer"
+        >
+          <h2 className="mb-3 text-2xl font-semibold">
+            Deploy{" "}
+            <span className="inline-block transition-transform group-hover:translate-x-1 motion-reduce:transform-none">
+              -&gt;
+            </span>
+          </h2>
+          <p className="m-0 max-w-[30ch] text-balance text-sm opacity-50">
+            Instantly deploy your Next.js site to a shareable URL with Vercel.
+          </p>
+        </a>
+      </div>
+    </main>
+  );
+}

nextjs/next.config.mjs

@@ -0,0 +1,4 @@
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+export default nextConfig;