diff --git a/src/config/chatAPI-config.ts b/src/config/chatAPI-config.ts index 11b2b76..677ce70 100644 --- a/src/config/chatAPI-config.ts +++ b/src/config/chatAPI-config.ts @@ -15,7 +15,8 @@ export const guidelines = { UNRELATED_QUESTION: `If the question is not related to the context, say this phrase EXACTLY '${ERROR_MESSAGES.NO_ANSWER}'`, LINKING: `DO NOT explicity mention the existence of the context provided, however, references can and should be made to the links provided in the context e.g '[0]'.`, FOLLOW_UP_QUESTIONS: 'If you have an answer, generate four relevant follow up questions, do not include introductory text about follow-up questions. Each question must be in this format: `--{{ what problems did segwit solve }}--` in a new line.', - USED_SOURCES: `Lastly, list all sources relevant in generating the answer in a list in this format '__sources__: [LINK_INDICES_HERE]'` + USED_SOURCES: `Lastly, list all sources relevant in generating the answer in a list in this format '__sources__: [LINK_INDICES_HERE]'`, + FALLBACK_INSTRUCTION: `You are an AI assistant providing helpful answers to user queries. Answer the user's question directly and concisely without using any external context or search results. You must rely on your own internal knowledge to generate the response.`, }; export const CONTEXT_WINDOW_MESSAGES = 6 diff --git a/src/pages/api/server.ts b/src/pages/api/server.ts index c42b021..9c8b42d 100644 --- a/src/pages/api/server.ts +++ b/src/pages/api/server.ts @@ -50,11 +50,11 @@ export default async function handler(req: Request) { const requesturl = req.url; const reqBody = await req.json(); - let esResults: any[] | null; + let esResults: any[] | null = null; let userQuery; const inputs = reqBody?.inputs; - const { query, author }: { query: string; author: string } = inputs; + const { query, author, fallback }: { query: string; author: string; fallback?: boolean } = inputs; if (!query) { return new Response( @@ -66,20 +66,22 @@ export default async function handler(req: Request) { const chatHistory = reqBody?.chatHistory ?? ([] as ChatHistory[]); try { - const fetchUrl = getNewUrl(requesturl, "/search"); + if (!fallback) { + const fetchUrl = getNewUrl(requesturl, "/search"); - const gptKeywords = await GPTKeywordExtractor([...chatHistory]); + const gptKeywords = await GPTKeywordExtractor([...chatHistory]); - esResults = await internalFetch({url: fetchUrl, query, author, keywords: gptKeywords}); + esResults = await internalFetch({ url: fetchUrl, query, author, keywords: gptKeywords }); - // FOR logging purposes - const loggedResultsURLs = esResults?.map(result => result?._source.url) - console.log(`query: ${query}\n gptKeywords: ${gptKeywords} \n results: ${loggedResultsURLs}`) + // FOR logging purposes + const loggedResultsURLs = esResults?.map(result => result?._source.url) + console.log(`query: ${query}\n gptKeywords: ${gptKeywords} \n results: ${loggedResultsURLs}`) - if (!esResults || !esResults.length) { - const error = createReadableStream(ERROR_MESSAGES.NO_ANSWER); - console.log(error); - return new Response(error); + if (!esResults || !esResults.length) { + const error = createReadableStream(ERROR_MESSAGES.NO_ANSWER); + console.log(error); + return new Response(error); + } } } catch (error) { console.log(error); @@ -90,7 +92,7 @@ export default async function handler(req: Request) { } try { - const result = await processInput(esResults, query, chatHistory); + const result = await processInput(esResults ?? undefined, query, chatHistory, fallback); return new Response(result); } catch (error: any) { const errMessage = error?.message diff --git a/src/service/chat/history.ts b/src/service/chat/history.ts index d008a99..295b03f 100644 --- a/src/service/chat/history.ts +++ b/src/service/chat/history.ts @@ -3,14 +3,20 @@ import { separateLinksFromApiMessage } from "@/utils/links"; import { CONTEXT_WINDOW_MESSAGES, guidelines } from "@/config/chatAPI-config"; import { ChatHistory } from "@/types"; -const buildSystemMessage = (question: string, context: string) => { +const buildSystemMessage = (question: string, context: string, fallback?: boolean) => { const { BASE_INSTRUCTION, NO_ANSWER, UNRELATED_QUESTION, FOLLOW_UP_QUESTIONS, LINKING, + FALLBACK_INSTRUCTION, } = guidelines; + + if (fallback) { + return `${FALLBACK_INSTRUCTION}`; + } + return `${BASE_INSTRUCTION}\n${NO_ANSWER}\n${UNRELATED_QUESTION}\n${context}\n${LINKING}\n${FOLLOW_UP_QUESTIONS}`; }; @@ -19,13 +25,15 @@ export const buildChatMessages = ({ context, oldContext, messages, + fallback, }: { question: string; context: string; oldContext?: string; messages: ChatHistory[]; + fallback?: boolean; }) => { - const systemMessage = buildSystemMessage(question, context); + const systemMessage = buildSystemMessage(question, context, fallback); return [ { role: "system", diff --git a/src/utils/openaiChat.ts b/src/utils/openaiChat.ts index 8ea332d..942606b 100644 --- a/src/utils/openaiChat.ts +++ b/src/utils/openaiChat.ts @@ -17,10 +17,12 @@ interface CustomContent { link: string; } + interface EnforceTokenParams { question: string; slicedTextWithLink: SummaryData[]; - chatHistory: ChatHistory[] + chatHistory: ChatHistory[]; + fallback?: boolean; } interface EnforceTokenLimitReturnType { @@ -50,11 +52,11 @@ function concatenateTextFields(data: string | ElementType[]): string { let elementArray: ElementType[]; // Check whether data is JSON string - if(typeof data === "string") { + if (typeof data === "string") { try { elementArray = JSON.parse(data); } - catch(e) { + catch (e) { // If it's not a JSON string. Then, consider the whole string as text. return data; } @@ -65,14 +67,14 @@ function concatenateTextFields(data: string | ElementType[]): string { // If data is an array of `ElementType` elementArray.forEach((element: ElementType) => { - if(element.type === "paragraph") { - concatenatedText += element.text + " "; - } else if(element.type === "heading") { - concatenatedText += "\n\n" + element.text + "\n\n"; + if (element.type === "paragraph") { + concatenatedText += element.text + " "; + } else if (element.type === "heading") { + concatenatedText += "\n\n" + element.text + "\n\n"; } }); - return concatenatedText.trim(); - } + return concatenatedText.trim(); +} function cleanText(text: string): string { @@ -95,83 +97,88 @@ const generateContextBlock = (summaries: SummaryData[]): string => { }; - async function SummaryGenerate( - link: SummaryData[], - messages: ChatHistory[], - retry: number = 0 - ): Promise> { - try { - const payload = { - model: process.env.OPENAI_MODEL, - messages, - temperature: 0.7, - top_p: 1.0, - frequency_penalty: 0.0, - presence_penalty: 1, - max_tokens: 700, - stream: true, - }; - const payloadJSON = JSON.stringify(payload) - - const response = await fetch(COMPLETION_URL,{ - headers: { - "Content-Type": "application/json", - Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ""}`, - }, - method: "POST", - body: payloadJSON, - } - ); - - if (!response.ok) { - return createReadableStream(ERROR_MESSAGES.NO_RESPONSE); - } +async function SummaryGenerate( + link: SummaryData[], + messages: ChatHistory[], + retry: number = 0, + fallback?: boolean +): Promise> { + try { + const payload = { + model: process.env.OPENAI_MODEL, + messages, + temperature: 0.7, + top_p: 1.0, + frequency_penalty: 0.0, + presence_penalty: 1, + max_tokens: 700, + stream: true, + }; + const payloadJSON = JSON.stringify(payload) + + const response = await fetch(COMPLETION_URL, { + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${process.env.OPENAI_API_KEY ?? ""}`, + }, + method: "POST", + body: payloadJSON, + } + ); + + if (!response.ok) { + return createReadableStream(ERROR_MESSAGES.NO_RESPONSE); + } - const encoder = new TextEncoder(); - const decoder = new TextDecoder(); - - const stream = new ReadableStream({ - async start(controller) { - function onParse(event: ParsedEvent | ReconnectInterval) { - if (event.type === "event") { - const data = event.data; - let text = "" - try { - if (data === "[DONE]") { - text = formatLinksToSourcesList(link) - const queue = encoder.encode(text); - controller.enqueue(queue); - controller.close() - return; + const encoder = new TextEncoder(); + const decoder = new TextDecoder(); + + const stream = new ReadableStream({ + async start(controller) { + function onParse(event: ParsedEvent | ReconnectInterval) { + if (event.type === "event") { + const data = event.data; + let text = "" + try { + if (data === "[DONE]") { + if (!fallback) { + text = formatLinksToSourcesList(link); + } else { + text = "\n\nDisclaimer: This response was generated without access to external sources and may be inaccurate." } - const jsonData = JSON.parse(data) - text = jsonData?.choices[0]?.delta?.content || '' - const queue = encoder.encode(text); controller.enqueue(queue); - } catch (e) { controller.close() + return; } + const jsonData = JSON.parse(data) + text = jsonData?.choices[0]?.delta?.content || '' + + const queue = encoder.encode(text); + controller.enqueue(queue); + } catch (e) { + controller.close() } } - const parser = createParser(onParse); - // https://web.dev/streams/#asynchronous-iteration - for await (const chunk of response.body as any) { - parser.feed(decoder.decode(chunk)); - } } - }) + const parser = createParser(onParse); + // https://web.dev/streams/#asynchronous-iteration + for await (const chunk of response.body as any) { + parser.feed(decoder.decode(chunk)); + } + } + }) - return stream + return stream - } catch (error) { - if (retry < 2) { - return SummaryGenerate(link, messages, retry + 1); - } else { - return createReadableStream(ERROR_MESSAGES.OVERLOAD); - } + } catch (error) { + if (retry < 2) { + return SummaryGenerate(link, messages, retry + 1, fallback); + } else { + return createReadableStream(ERROR_MESSAGES.OVERLOAD); } } +} function removeDuplicatesByID(arr: CustomContent[]): CustomContent[] { const seen = new Set(); @@ -200,36 +207,39 @@ function formatLinksToSourcesList( export async function processInput( searchResults: Result[] | undefined, question: string, - chatHistory: ChatHistory[] + chatHistory: ChatHistory[], + fallback?: boolean ) { try { - if (!searchResults?.length) { + if (!searchResults?.length && !fallback) { let output_string: string = ERROR_MESSAGES.NO_ANSWER; return createReadableStream(output_string); } else { const intermediateContent: CustomContent[] = [] - for (const result of searchResults) { - let { _source: source} = result - const isQuestionOnStackExchange = - source.type === "question" && - source.url.includes("stackexchange"); - if (!isQuestionOnStackExchange) { - const isMarkdown = source.body_type === "markdown"; - const snippet = isMarkdown - ? concatenateTextFields(source.body) - : source.body; - intermediateContent.push({ - title: source.title, - snippet: snippet, - link: source.url, - }); + if (searchResults) { + for (const result of searchResults) { + let { _source: source } = result + const isQuestionOnStackExchange = + source.type === "question" && + source.url.includes("stackexchange"); + if (!isQuestionOnStackExchange) { + const isMarkdown = source.body_type === "markdown"; + const snippet = isMarkdown + ? concatenateTextFields(source.body) + : source.body; + intermediateContent.push({ + title: source.title, + snippet: snippet, + link: source.url, + }); + } } } const deduplicatedContent = removeDuplicatesByID(intermediateContent); - if (!deduplicatedContent.length) { + if (!deduplicatedContent.length && !fallback) { throw new Error(ERROR_MESSAGES.NO_ANSWER) } @@ -240,9 +250,9 @@ export async function processInput( }) ); - const {messages, slicedTextWithLink: finalSources} = enforceTokenLimit({question, slicedTextWithLink, chatHistory}) + const { messages, slicedTextWithLink: finalSources } = enforceTokenLimit({ question, slicedTextWithLink, chatHistory, fallback }) - const summary = await SummaryGenerate(finalSources, messages); + const summary = await SummaryGenerate(finalSources, messages, 0, fallback); return summary } } catch (error: any) { @@ -251,14 +261,18 @@ export async function processInput( } } -function enforceTokenLimit ({question, slicedTextWithLink, chatHistory}: EnforceTokenParams): EnforceTokenLimitReturnType { - const messages = buildChatMessages({question, context: generateContextBlock(slicedTextWithLink), messages: chatHistory }) - let tokenLength = promptTokensEstimate({messages}) - console.log({tokenLength}) +function enforceTokenLimit({ question, slicedTextWithLink, chatHistory, fallback }: EnforceTokenParams): EnforceTokenLimitReturnType { + const context = fallback ? "" : generateContextBlock(slicedTextWithLink); + const messages = buildChatMessages({ question, context, messages: chatHistory, fallback }) + let tokenLength = promptTokensEstimate({ messages }) + console.log({ tokenLength }) if (tokenLength > TOKEN_UPPER_LIMIT) { - slicedTextWithLink.pop() - chatHistory.length>4 && chatHistory.shift() - return enforceTokenLimit({question, slicedTextWithLink, chatHistory}) + if (slicedTextWithLink.length > 0) { + slicedTextWithLink.pop() + } else { + chatHistory.length > 4 && chatHistory.shift() + } + return enforceTokenLimit({ question, slicedTextWithLink, chatHistory, fallback }); } - return {slicedTextWithLink, messages, tokenLength} + return { slicedTextWithLink, messages, tokenLength } } \ No newline at end of file