Crawl Pinecone docs and marketing site instead of demo URLs

src/app/api/checkIndex/route.ts

@@ -1,11 +1,33 @@
 import { Pinecone } from '@pinecone-database/pinecone';
 import { NextResponse } from "next/server";
+import { ServerlessSpecCloudEnum } from '@pinecone-database/pinecone/dist/pinecone-generated-ts-fetch';
+import {
+    PINECONE_REGION,
+    PINECONE_CLOUD,
+    DIMENSION
+} from "@/utils/serverlessConfig"
 
 export async function POST() {
     // Instantiate a new Pinecone client
     const pinecone = new Pinecone();
     // Select the desired index
     const indexName = process.env.PINECONE_INDEX!;
+
+    // Create the index if it doesn't already exist, 
+    // but do not throw an error if it does already exist
+    await pinecone.createIndex({
+        name: indexName,
+        dimension: Number(DIMENSION),
+        suppressConflicts: true,
+        waitUntilReady: true,
+        spec: {
+            serverless: {
+                region: PINECONE_REGION,
+                cloud: PINECONE_CLOUD as ServerlessSpecCloudEnum,
+            }
+        }
+    })
+
     const index = pinecone.Index(indexName);
 
     // Use the custom namespace, if provided, otherwise use the default

src/app/api/crawl/seed.ts

@@ -6,16 +6,18 @@ import { Pinecone, PineconeRecord } from "@pinecone-database/pinecone";
 import { ServerlessSpecCloudEnum } from '@pinecone-database/pinecone/dist/pinecone-generated-ts-fetch';
 import md5 from "md5";
 import { Crawler, Page } from "./crawler";
+import {
+  PINECONE_REGION,
+  PINECONE_CLOUD,
+  DIMENSION
+} from "@/utils/serverlessConfig"
 
 interface SeedOptions {
   splittingMethod: string
   chunkSize: number
   chunkOverlap: number
 }
 
-const PINECONE_REGION = process.env.PINECONE_REGION || 'us-west-2'
-const PINECONE_CLOUD = process.env.PINECONE_CLOUD || 'aws'
-
 type DocumentSplitter = RecursiveCharacterTextSplitter | MarkdownTextSplitter
 
 async function seed(url: string, limit: number, indexName: string, options: SeedOptions) {
@@ -46,8 +48,11 @@ async function seed(url: string, limit: number, indexName: string, options: Seed
     if (!indexExists) {
       await pinecone.createIndex({
         name: indexName,
-        dimension: 1536,
+        dimension: Number(DIMENSION),
+        // Wait until the index is ready to return, if it needs to be created
         waitUntilReady: true,
+        // If an index with the target name already exists, do not throw an error
+        suppressConflicts: true,
         spec: {
           serverless: {
             region: PINECONE_REGION,
@@ -127,7 +132,4 @@ async function prepareDocument(page: Page, splitter: DocumentSplitter): Promise<
   });
 }
 
-
-
-
 export default seed;

src/app/components/Sidebar/urls.ts

@@ -1,19 +1,14 @@
 export const urls = [
   {
-    url: "https://www.wired.com/story/fast-forward-toyota-robots-learning-housework/",
-    title: "Toyota's Robots are Learning Housework",
+    url: "https://docs.pinecone.io",
+    title: "Pinecone's official documentation",
     seeded: false,
     loading: false,
   },
   {
-    url: "https://www.wired.com/story/synthetic-data-is-a-dangerous-teacher/",
-    title: "Synthetic Data Is a Dangerous Teacher",
+    url: "https://www.pinecone.io",
+    title: "Pinecone's marketing site",
     seeded: false,
     loading: false,
-  },
-  {
-    url: "https://www.wired.com/story/staying-one-step-ahead-of-hackers-when-it-comes-to-ai/",
-    title: "Staying Ahead of Hackers When It Comes to AI",
-    seeded: false,
-    loading: false,
-  }]
+  }
+]

src/app/page.tsx

@@ -7,14 +7,21 @@ import type { PineconeRecord } from "@pinecone-database/pinecone";
 import React, { useEffect, useState } from "react";
 import { FaGithub } from 'react-icons/fa';
 import AppContext from "./appContext";
+import checkRequiredEnvVars from "@/utils/requiredEnvVars"
 
 const Page: React.FC = () => {
   const [context, setContext] = useState<{ context: PineconeRecord[] }[] | null>(null);
   const { totalRecords, refreshIndex } = useRefreshIndex();
 
   useEffect(() => {
     if (totalRecords === 0) {
-      refreshIndex()
+      try {
+        refreshIndex()
+      } catch (error: unknown) {
+        if (error instanceof Error) {
+          throw new Error(error.message)
+        }
+      }
     }
   }, [refreshIndex, totalRecords])
 

src/app/utils/requiredEnvVars.ts

@@ -0,0 +1,26 @@
+/**                                                                                                                                                        
+    * Checks required environment variables and throws an error if any are                                                                                    
+ missing.                                                                                                                                                     
+    * @throws {Error} with the names of the missing environment variables.                                                                                    
+    */
+function checkRequiredEnvVars(): void {
+  // Define the required environment variables                            
+  const requiredVars: string[] = [
+    'PINECONE_INDEX',
+    'PINECONE_API_KEY',
+    'PINECONE_CLOUD',
+    'OPENAI_API_KEY'
+  ];
+
+  // Collect the names of any missing environment variables               
+  const missingVars: string[] = requiredVars.filter(varName =>
+    !process.env[varName]);
+
+  // If there are any missing variables, throw an error listing them      
+  if (missingVars.length > 0) {
+    throw new Error(`Missing required environment variables:              
+  ${missingVars.join(', ')}`);
+  }
+}
+
+export default checkRequiredEnvVars;            

src/app/utils/serverlessConfig.js

@@ -0,0 +1,10 @@
+const PINECONE_REGION = process.env.PINECONE_REGION || 'us-west-2'
+const PINECONE_CLOUD = process.env.PINECONE_CLOUD || 'aws'
+const DIMENSION = process.env.DIMENSION || 1536;
+
+export {
+  PINECONE_REGION,
+  PINECONE_CLOUD,
+  DIMENSION
+}
+

src/middleware.ts

@@ -1,12 +1,14 @@
 import type { NextRequest } from 'next/server';
 import { NextResponse } from 'next/server';
+import checkRequiredEnvVars from '@/utils/requiredEnvVars';
 
-export function middleware(request: NextRequest) {
-    const requiredEnvVars = ['OPENAI_API_KEY', 'PINECONE_API_KEY', 'PINECONE_REGION', 'PINECONE_INDEX'];
-    requiredEnvVars.forEach(envVar => {
-        if (!process.env[envVar] && !process.env.CI) {
-            throw new Error(`${envVar} environment variable is not defined`);
+export function middleware(_request: NextRequest) {
+    try {
+        checkRequiredEnvVars();
+    } catch (error: unknown) {
+        if (error instanceof Error) {
+            return new NextResponse(JSON.stringify({ error: error.message }), { status: 400 })
         }
-    });
+    }
     return NextResponse.next()
-}
+}