import { HttpException, HttpStatus, Injectable, UnauthorizedException, } from '@nestjs/common'; // import { ModelService } from 'src/model/model.service'; import { PDFLoader } from '@langchain/community/document_loaders/fs/pdf'; import { DocxLoader } from '@langchain/community/document_loaders/fs/docx'; import { RecursiveCharacterTextSplitter } from 'langchain/text_splitter'; import { ModelService } from '../model/model.service'; import { CommonService } from '../common/common.service'; import moment from 'moment'; import * as path from "path"; import * as fs from "fs"; import * as dto from './dto/index'; import { GoogleGenerativeAI } from '@google/generative-ai'; import { Pinecone } from '@pinecone-database/pinecone'; import { Types } from 'mongoose' import { config } from 'dotenv' config() import axios from 'axios'; import * as cheerio from 'cheerio'; import { MilvusClient, DataType } from '@zilliz/milvus2-sdk-node'; import OpenAI from 'openai'; import Anthropic from '@anthropic-ai/sdk'; import { TextToSpeechService } from '../services/tts.service'; import { CallService } from '../call/call.service'; // const collectionName = "khowledge_base_1"; // Dev server // const collectionName = "khowledge_base_3"; // local type model_result = { answer: string, token_uses: number } @Injectable() export class KnowledgeService { private genAI: GoogleGenerativeAI; private pineconeClient: Pinecone; private milvusClient: any; private vectorDimension: number; private genAIEmbed: any; private openAIEmbed: any; private genAImodel: any; private provider: string; private collectionName: string; constructor( private readonly Model: ModelService, private readonly CommonService: CommonService, private readonly CallService: CallService, ) { this.genAI = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY); this.pineconeClient = new Pinecone({ apiKey: process.env.PINECONE_API_KEY, }); this.milvusClient = new MilvusClient({ address: process.env.MILVUS_DB_URL, // token: "qixs:Qixs@2025" }); this.openAIEmbed = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); this.vectorDimension = 728; this.genAIEmbed = this.genAI.getGenerativeModel({ model: 'embedding-001', }); this.collectionName = process.env.MILVUS_COLLECTION_NAME; this.setProvider('default'); } setProvider(provider: string) { console.log('#############', provider) this.provider = provider.toLowerCase(); switch (provider.toLowerCase()) { case 'gimini': this.genAImodel = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY); break; case 'chatgpt': this.genAImodel = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); break; case 'claud': this.genAImodel = new Anthropic({ apiKey: process.env.ANTHROPIC_API_KEY }); break; case 'deepseek': this.genAImodel = new OpenAI({ baseURL: 'https://api.deepseek.com/v1', apiKey: process.env.DEEPSEEK_API_KEY }); break; case 'default': default: this.genAImodel = new GoogleGenerativeAI(process.env.GOOGLE_API_KEY); break; } } async useMilvusDatabase() { try { // const data = await this.milvusClient.createDatabase({ db_name: 'dialai' }); await this.milvusClient.useDatabase({ db_name: 'dialai' }); } catch (error) { throw error; } }; async dropCollection() { try { const hasCollection = await this.milvusClient.hasCollection({ collection_name: this.collectionName, }); if (!hasCollection.value) { console.log(`Collection "${this.collectionName}" does not exist.`); return; } // Drop the collection const response = await this.milvusClient.dropCollection({ collection_name: this.collectionName, }); if (response.error_code !== 0) { throw new Error(`Failed to drop collection: ${response.reason}`); } console.log(`Collection "${this.collectionName}" has been dropped successfully.`); } catch (error) { console.error('Error dropping collection:', error.message); throw error; } } async knowledge_base_create(body: dto.Knowledge.knowledgeBaseCreate, req: any) { try { let { _id: vendor_id } = req.user_data let { name } = body let user_workspace = await this.Model.UserWorkspaceModel.findOne({ vendor_id: new Types.ObjectId(vendor_id) }) if (!user_workspace) throw new HttpException('You are not in workspace', HttpStatus.BAD_REQUEST) return await this.Model.knowledgeBaseModel.create({ workspace_id: user_workspace?.workspace_id, name: name }) } catch (error) { throw error } } async knowledge_base_get(body: dto.Knowledge.getKnowledgeBase, req: any) { try { let { _id: vendor_id } = req.user_data let { pagination, limit } = body let user_workspace = await this.Model.UserWorkspaceModel.findOne({ vendor_id: new Types.ObjectId(vendor_id) }) if (!user_workspace) throw new HttpException('You are not in workspace', HttpStatus.BAD_REQUEST) let options = await this.CommonService.set_options(pagination, limit) let data = await this.Model.knowledgeBaseModel.find({ workspace_id: user_workspace?.workspace_id }, {}, options) let count = await this.Model.knowledgeBaseModel.countDocuments({ workspace_id: user_workspace?.workspace_id }) return { count: count, data: data } } catch (error) { throw error } } async knowledge_base_detail(knowledge_base_id: string, req: any) { try { let { _id: vendor_id } = req.user_data let data = await this.Model.knowledgeBaseModel.findOne({ _id: knowledge_base_id }, {}, { lean: true }) return data } catch (error) { throw error } } async knowledge_base_documents(knowledge_base_id: string, body: dto.Knowledge.getKnowledgeBaseDocument, req: any) { try { let { _id: vendor_id } = req.user_data let { pagination, limit, search } = body let options = await this.CommonService.set_options(pagination, limit) let query = { knowledge_base_id: new Types.ObjectId(knowledge_base_id), ...(search && { name: { $regex: search, $options: "i" } }) } let data = await this.Model.documentsModel.find(query, {}, options) let count = await this.Model.documentsModel.countDocuments(query) return { count: count, data: data } } catch (error) { throw error } } async deleteByDocumentIdsMilvus(documentIds: string | string[]) { try { if (!Array.isArray(documentIds)) { documentIds = [documentIds]; } const expr = `documentId in [${documentIds.map(id => `'${id}'`).join(', ')}]`; const response = await this.milvusClient.deleteEntities({ collection_name: this.collectionName, expr: expr, }); console.log(`Deletion response for documentIds "${documentIds}":`, response?.delete_cnt); } catch (error) { console.error(`Error deleting data for documentIds "${documentIds}":`, error.message); } } async document_delete(document_id: string, req: any) { try { const document = await this.Model.documentsModel.findOne({ _id: new Types.ObjectId(document_id) }, {}, { lean: true }) // let index_data = await this.pineconeClient.describeIndex(process.env.PINECONE_INDEX_NAME) // // console.log("index_data", index_data); // const index = await this.pineconeClient.Index(process.env.PINECONE_INDEX_NAME, index_data?.host); // const ns = index.namespace("") // ns.deleteMany(document?.document_ids) /** Delete from Milvus */ await this.deleteByDocumentIdsMilvus(document_id); let data = await this.Model.documentsModel.deleteOne({ _id: new Types.ObjectId(document_id) }) return { message: "success" } } catch (error) { throw error } } async knowledge_base_delete(knowledge_base_id: string, req: any) { try { let { _id: vendor_id } = req.user_data const document_ids = await this.knowledge_base_document_ids(knowledge_base_id) // let index_data = await this.pineconeClient.describeIndex(process.env.PINECONE_INDEX_NAME) // // console.log("index_data", index_data); // const index = await this.pineconeClient.Index(process.env.PINECONE_INDEX_NAME, index_data?.host); // // console.log("index_data", index); // const ns = index.namespace("") // ns.deleteMany(document_ids) console.log('document_ids', document_ids) /** Delete from Milvus */ await this.deleteByDocumentIdsMilvus(document_ids); let data = await this.Model.documentsModel.deleteMany({ knowledge_base_id: new Types.ObjectId(knowledge_base_id) }) await this.Model.knowledgeBaseModel.deleteOne({ _id: new Types.ObjectId(knowledge_base_id) }) return { message: "success" } } catch (error) { throw error } } async knowledge_base_document_ids(knowledge_base_id: string) { try { let document = await this.Model.documentsModel.find({ knowledge_base_id: new Types.ObjectId(knowledge_base_id) }, {}, { lean: true }) let document_id: string[] = []; // for (let i = 0; i < document.length; i++) { // const { document_ids } = document[i]; // document_id = [...document_ids, ...document_id] // } document.map((doc: any) => document_id.push(doc?._id)) return document_id } catch (error) { throw error } } async pdfUpload(file: any, knowledge_base_id: string) { let folder_path = path.resolve( __dirname + '/assets/all_pdfs/' + file.originalname ); fs.writeFileSync(folder_path, file.buffer); let loader const fileExtension = path.extname(folder_path).toLowerCase(); if (fileExtension === '.pdf') { loader = new PDFLoader(folder_path); } else if (fileExtension === '.docx') { loader = new DocxLoader(folder_path); } else { throw new Error(`Unsupported file type: ${fileExtension}`); } const docs = await loader.load(); const combinedData = docs.map(page => page.pageContent).join(' '); // console.log("combinedData",combinedData); let document = await this.Model.documentsModel.create({ type: "PDF", name: file.originalname, document_title: file.originalname, content: combinedData, knowledge_base_id: new Types.ObjectId(knowledge_base_id) }) try { const splitDocs = await this.splitDocs(folder_path); const batchSize = 1; let data_to_push = []; for (let i = 0; i < splitDocs.length; i++) { const batch = splitDocs.slice(i, i + batchSize); for (let index = 0; index < batch.length; index++) { const element = batch[index]; data_to_push.push(this.addDataToPinecone(element.pageContent, document?._id?.toString(), knowledge_base_id)); if (data_to_push.length === 10) { let resolve = await Promise.all(data_to_push); data_to_push = []; } } } await Promise.all(data_to_push) fs.unlinkSync(folder_path); return splitDocs; } catch (error) { fs.unlink(folder_path, () => { }); await this.Model.documentsModel.deleteOne({ _id: document?._id }); throw error; } } async splitDocs(folder_path: any) { try { const fileExtension = path.extname(folder_path).toLowerCase(); let loader; if (fileExtension === '.pdf') { loader = new PDFLoader(folder_path); } else if (fileExtension === '.docx') { loader = new DocxLoader(folder_path); } else { throw new Error(`Unsupported file type: ${fileExtension}`); } const docs = await loader.load(); console.log("docs1", docs); const splitter = new RecursiveCharacterTextSplitter({ chunkSize: 20000, chunkOverlap: 200, }); const splitDocs = await splitter.splitDocuments(docs); console.log('splitDocs', splitDocs.length); console.log('splitDocs', splitDocs[0].pageContent.length); return splitDocs; } catch (error) { throw error; } } async addDataToPinecone(content: any, document_id: string, knowledge_base_id: string) { try { content = content.trim().replace(/\s+/g, ' '); const embedding = await this.embedContent(content); let id = `docs ${+new Date()}` let vector = [ { id: id, values: embedding, metadata: { text: content, document_id: document_id, ...(knowledge_base_id && { knowledge_base_id: knowledge_base_id }) }, }, ]; this.addDocument(vector); this.Model.documentsModel.findOneAndUpdate({ _id: new Types.ObjectId(document_id) }, { $addToSet: { document_ids: id } }) return embedding; } catch (error) { throw error; } } async addDataToMilvus(content: any, document_id: string, knowledge_base_id: string) { try { content = content.trim().replace(/\s+/g, ' '); const embedding = await this.embedContent(content); let id = `docs_${+new Date()}` let vector = [ { id: id, values: embedding, // metadata: { // text: content, // document_id: document_id, // ...(knowledge_base_id && { knowledge_base_id: knowledge_base_id }) // }, metadata: content, documentId: document_id }, ]; this.addDocument(vector); await this.Model.documentsModel.findOneAndUpdate({ _id: new Types.ObjectId(document_id) }, { $addToSet: { document_ids: id } }) return embedding; } catch (error) { throw error; } } async embedContent(content: string) { try { content = content.trim().replace(/\s+/g, ' '); let embedding: any; embedding = await this.openAIEmbed.embeddings.create({ model: "text-embedding-3-small", input: content, }); embedding = embedding.data[0].embedding; if (embedding.length > this.vectorDimension) { embedding = embedding.slice(0, this.vectorDimension); } else if (embedding.length < this.vectorDimension) { const padding = new Array(this.vectorDimension - embedding.length).fill(0); embedding = embedding.concat(padding); } return embedding; } catch (error) { throw error; } } // async embedContent(content: string) { // try { // // // For embeddings, use the Text Embeddings model // // const model = this.genAI.getGenerativeModel({ // // model: 'text-embedding-004', // // }); // // // const text = 'The quick brown fox jumps over the lazy dog.'; // // const result = await model.embedContent(content); // // const embedding = result.embedding; // // // console.log(embedding.values.length); // // return embedding.values; // content = content.trim().replace(/\s+/g, ' '); // const model = 'multilingual-e5-large'; // const embeddings = await this.pineconeClient.inference.embed( // model, // [content], // { inputType: 'passage', truncate: 'END' } // ); // // console.log("embeddings", embeddings[0]?.values); // return embeddings[0]?.values // } catch (error) { // throw error; // } // } async addDocument(vectors: any) { try { // const index = this.pineconeClient.Index(process.env.PINECONE_INDEX_NAME); // const res = await index.upsert(vectors); await this.createCollectionIfNotExists(); const response = await this.milvusClient.insert({ collection_name: this.collectionName, data: vectors, }); console.log('response', response?.status?.error_code); return true; } catch (error) { throw error; } } async textUpload(text: string, title: string, knowledge_base_id: string) { let document = await this.Model.documentsModel.create({ type: "TEXT", name: title, document_title: title, content: text, ...(knowledge_base_id && { knowledge_base_id: new Types.ObjectId(knowledge_base_id) }) }) try { text = text.trim().replace(/<\/?[^>]+(>|$)/g, " "); let embeddings = this.getChunks(text); let data_to_push = []; for (let i = 0; i < embeddings.length; i++) { const content = embeddings[i]; let embedding = this.addDataToMilvus(content, document?._id?.toString(), knowledge_base_id); data_to_push.push(embedding); if (data_to_push.length === 10) { let resolve = await Promise.all(data_to_push); data_to_push = []; } } await Promise.all(data_to_push); return { message: "success" } } catch (error) { await this.Model.documentsModel.deleteOne({ _id: document?._id }) throw error; } } getChunks = (text: string): string[] => { const chunks: string[] = []; let start = 0; const maxChunkSize = 9000; while (start < text.length) { let end = start + maxChunkSize; // Ensure we don’t cut in the middle of a word if (end < text.length) { const lastSpace = text.lastIndexOf(' ', end); if (lastSpace > start) end = lastSpace; } chunks.push(text.slice(start, end)); start = end + 1; } return chunks; }; async getAnswer(query: string, chat_history: string, agent_id: string) { try { console.time('Total time of vector db'); // let context = await this.getDocument(query, agent_id); let context = await this.getDataFromMilvus(query, agent_id); console.log('context', context) console.timeEnd('Total time of vector db'); const prompt = ` You are an inbound custmore care represntative for henceforth solutions. You have a youthful and cheery personality. Keep your responses as brief as possible but make every attempt to keep the coustmer in the chat bot without being rude. Don't make assumptions Ask for clarification if a user request is ambiguous. You are given a reference context that may contain relevant information for answering the user's question. Provide a complete and clear answer, using relatable examples and analogies where possible to make the response easy to understand. If you cannot find relevant information in the provided context to answer the question, please apologize politely and explain that you don't have enough information to provide a proper answer. If the context is helpful, integrate it into your answer and summarize it and don't give the context in the answer; if it is not directly relevant, feel free to exclude it. Important Note please give answer in just 30 word very short and clear answer Question: ${query} Context: ${context} this is our chat history answer accordingly ${chat_history} `.trim(); const model = this.genAI.getGenerativeModel({ model: 'gemini-1.5-flash', generationConfig: { // candidateCount: 1, // stopSequences: ["x"], maxOutputTokens: 200, temperature: 0.7, }, }); console.log("getAnswer(query: string, chat_history: string, agent_id: string)") console.time('time in gemini response'); const result = await model.generateContent(prompt); console.timeEnd('time in gemini response'); return { answer: result.response.text(), }; } catch (error) { throw error; } } // async getDocument(query: string, agent_id: string) { // try { // console.time('Embedding generation'); // const vector = await this.embedContent(query); // console.timeEnd('Embedding generation'); // console.time('Vector DB response'); // let document_ids = await this.getKnowledgeBaseDocuments(agent_id) // const index = this.pineconeClient.Index(process.env.PINECONE_INDEX_NAME); // const res = await index.query({ // vector: vector, // topK: 5, // includeMetadata: true, // // includeValues: true, // ...(document_ids.length && { filter: { document_id: { "$in": document_ids } } }) // }); // // console.log("res",res); // console.timeEnd('Vector DB response'); // for (let i = 0; i < res.matches.length; i++) { // const element = res.matches[i]; // } // const context = res.matches // .map((match) => match.metadata.text) // .join('\n\n'); // return context; // } catch (error) { // throw error; // } // } async getKnowledgeBaseDocuments(agent_id: string) { try { let agent_data = await this.Model.AgentModel.findOne({ _id: new Types.ObjectId(agent_id) }, { knowledge_base_id: 1, _id: 1 }, { lean: true }) if (!agent_data) throw new HttpException("Agent not found!", HttpStatus.BAD_REQUEST) if (!agent_data?.knowledge_base_id) { console.error('knowledge_base_id is not provided or invalid'); return; } const documents = await this.Model.documentsModel.find({ knowledge_base_id: new Types.ObjectId(String(agent_data.knowledge_base_id)), }, { _id: 1 }, { lean: true }); console.log('Documents found: ', documents.length); let data = documents.map((elem) => { return elem?._id.toString() }) return data } catch (error) { throw error; } } // async scrapAndUpload(url: string, title: string, knowledge_base_id: string) { // var document // try { // const { data: html } = await axios.get(url); // const $ = cheerio.load(html); // const textContent = $('body').text().trim(); // document = await this.Model.documentsModel.create({ // type: "TEXT", // name: title, // document_title: title, // content: textContent, // ...(knowledge_base_id && { knowledge_base_id: new Types.ObjectId(knowledge_base_id) }) // }) // let embeddings = this.getChunks(textContent); // let data_to_push = []; // for (let i = 0; i < embeddings.length; i++) { // const content = embeddings[i]; // let embedding = this.addDataToMilvus(content, document?._id.toString(), knowledge_base_id); // data_to_push.push(embedding); // if (data_to_push.length === 10) { // let resolve = await Promise.all(data_to_push); // // console.log('resolve', resolve[10]); // data_to_push = []; // } // // console.log('i', i); // } // await Promise.all(data_to_push); // return "Document processed successfully" // } catch (error) { // await this.Model.documentsModel.deleteOne({ _id: document?._id }) // throw error; // } // } async scrapAndUpload(url: string, title: string, knowledge_base_id: string, depth: number = 1, visited = new Set()) { let document; if (depth === 0 || visited.has(url)) return; try { console.log('url =======> ', url) const { data: html } = await axios.get(url); const $ = cheerio.load(html); $('style').remove(); $('[style]').removeAttr('style'); $('script').remove(); const textContent = $('body').text().trim(); console.log(`Scraping ${url} at depth ${depth}`); if (!textContent) { throw new Error(`No meaningful content found at URL: ${url}`); } document = await this.Model.documentsModel.create({ type: "TEXT", name: title, document_title: `${title}`, content: textContent.trim().replace(/\s+/g, ' '), ...(knowledge_base_id && { knowledge_base_id: new Types.ObjectId(knowledge_base_id) }), }); const chunks = this.getChunks(textContent); await this.uploadEmbeddings(chunks, document._id.toString(), knowledge_base_id); let extract_urls = []; if (depth > 1) { const links = this.extractLinks(url, $); extract_urls = links; for (const link of links) { title = link; await this.scrapAndUpload(link, title, knowledge_base_id, depth - 1); } } return extract_urls // "Document processed successfully"; } catch (error) { if (document?._id) { await this.Model.documentsModel.deleteOne({ _id: document._id }); } console.error(`Error processing document "${title}" from URL "${url}":`, error.message); throw error; } } // extractLinks(url: string, $: cheerio.CheerioAPI): string[] { // const links: string[] = []; // $('a').each((_, element) => { // let href = $(element).attr('href'); // if (href && !href.startsWith('#')) { // const absoluteLink = new URL(href, url).href; // links.push(absoluteLink); // } // }); // console.log('links', "LENGTH", links.length) // return links; // } extractLinks(url: string, $: cheerio.CheerioAPI): string[] { const links: string[] = []; $('a').each((_, element) => { let href = $(element).attr('href'); if (href && !href.startsWith('#')) { const absoluteLink = new URL(href, url).href; if (absoluteLink.startsWith(url)) { links.push(absoluteLink); } } }); console.log('links', links.length); return links; } async uploadEmbeddings(chunks: string[], documentId: string, knowledge_base_id: string) { const batchSize = 10; const embeddingPromises: any[] = []; for (const chunk of chunks) { embeddingPromises.push( this.addDataToMilvus(chunk, documentId, knowledge_base_id) ); if (embeddingPromises.length === batchSize) { await Promise.all(embeddingPromises); embeddingPromises.length = 0; } } if (embeddingPromises.length > 0) { await Promise.all(embeddingPromises); } } // Helper methods private async getChatGPTAnswer(messages: any) { const params = { model: 'gpt-4o-mini', messages, max_tokens: 200, temperature: 0.7, }; const response: OpenAI.Chat.ChatCompletion = await this.genAImodel.chat.completions.create(params); const token_uses: number = response.usage.total_tokens; // return response.choices[0].message.content, console.log('response GPT ==>>>>', response) return { answer: response.choices[0].message.content, token_uses: token_uses }; } private async getGeminiAnswer(prompt: string) { const model = this.genAImodel.getGenerativeModel({ model: 'gemini-1.5-flash', generationConfig: { maxOutputTokens: 200, temperature: 0.7 }, }); const result = await model.generateContent(prompt); const token_uses: number = result.response.usageMetadata.totalTokenCount; // return result.response.text(); console.log('result.response', result.response) return { answer: result.response.text(), token_uses: token_uses } } private async getClaudAnswer(prompt: string) { const stream = this.genAImodel.messages.stream({ model: 'claude-3-5-sonnet-latest', max_tokens: 200, temperature: 0.7, messages: [ { role: 'user', content: prompt, }, ], }); const message = await stream.finalMessage(); const token_uses: number = message.usage.total_tokens; console.log(message); return message; // return new Promise((resolve, reject) => { // let responseText = ''; // stream.on('text', (text) => { // responseText += text; // }); // stream.on('end', () => { // resolve(responseText); // }); // stream.on('error', (error) => { // reject(error); // }); // }); } getTokenCounts = (sentence: string) => { const tokens = sentence.match(/\w+|[?,]/g); return tokens.length } /** * THIS FUNCTION USED IN AI CHAT/ CHAT-BOT * @param query * @param chat_history * @param agent_id * @returns { answer } */ async getAnswerKnowledgeBaseChat(query: string, chat_history: any, agent_id: string, subscription_id: any, chat_token_limit_left: number, user_details: any) { try { console.time("AI RESPONSE TIME"); const [context, agent] = await Promise.all([ this.getDataFromMilvus(query, agent_id), this.get_agent_details(agent_id), ]); this.setProvider(agent?.ai_model); console.log('MODEL:', this.provider); const formattedContext = context.map((item: any, index: number) => ( `Document ${index + 1}:\n Score: ${item.score}\n Document ID: ${item.documentId}\n Metadata: ${item.metadata || 'N/A'}` )).join('\n'); const contextString = `Relevant Documents:\n${formattedContext}`.trim(); console.log('agent.chat_prompt', agent.chat_prompt) console.log('Question', query) const prompt = ` ${agent.chat_prompt} Question: ${query} Context: ${contextString} This is our chat history; answer accordingly: ${chat_history} and the user name is ${user_details.name} & email is ${user_details.email} `.trim(); console.log('prompt final ===========================>>>>>', prompt) console.log('chat_history ##################', chat_history) const messages = [ { role: 'system', content: agent.call_prompt }, { role: 'system', content: contextString }, { role: 'assistant', content: chat_history }, { role: 'user', content: query }, ]; const providerHandlers = { chatgpt: () => this.getChatGPTAnswer(messages), // chatgpt: () => this.getGeminiAnswer(prompt), gemini: () => this.getGeminiAnswer(prompt), claud: () => this.getClaudAnswer(prompt), deepseek: () => this.getClaudAnswer(prompt), }; const { answer, token_uses } = await providerHandlers[this.provider](); console.timeEnd("AI RESPONSE TIME"); console.log("TOKEN_AVAL :", chat_token_limit_left, `\nTOKEN USES (${this.provider}):`, token_uses) await this.Model.subscriptionModel.findOneAndUpdate( { _id: subscription_id }, { chat_token_limit_left: (chat_token_limit_left - token_uses) }, { new: true } ); return { answer }; } catch (error) { console.error("Error:", error); throw error; } } // USED THIS FUNCTION ONLY IN TELE-PHONE CALL async getAnswerKnowledgeBaseChatGPT(query: string, chat_history: string, agent_id: string) { try { let context = await this.getDataFromMilvus(query, agent_id); const agent = await this.get_agent_details(agent_id); this.setProvider(agent?.ai_model); const formattedContext = context.map((item: any, index: number) => { return `Document ${index + 1}:\n Score: ${item.score}\n Document ID: ${item.documentId}\n Metadata: ${item.metadata || 'N/A'}\n`; }).join('\n'); const contextString = ` Relevant Documents: ${formattedContext} `.trim(); const prompt = ` ${agent.call_prompt} Question: ${query} Context: ${contextString} this is our chat history answer accordingly ${chat_history} `.trim(); const params = { model: 'gpt-4o-mini', messages: [ { role: 'system', content: agent.call_prompt }, { role: 'user', content: query }, { role: 'system', content: contextString }, { role: 'assistant', content: chat_history }, ], max_tokens: 200, temperature: 0.7, }; // console.timeEnd("GET TIME") let response: { answer: string; token_uses: number }; console.time("AI RESPONSE TIME") console.log('provider ==========>> ', this.provider) switch (this.provider) { case "chatgpt": response = await this.getChatGPTAnswer(params); break; case "gemini": response = await this.getGeminiAnswer(prompt); break; case "claud": response = await this.getClaudAnswer(prompt); break; case "deepseek": response = await this.getClaudAnswer(prompt); break; default: throw new Error(`Unsupported provider: ${this.provider}`); } const { answer, token_uses } = response; console.timeEnd("AI RESPONSE TIME") return { answer, }; } catch (error) { throw error; } } // streamin resp for call only async getAnswerKnowledgeBaseChatLangChain(query: string, chat_history: string, agent_id: string) { try { const ttsService = new TextToSpeechService(); let context = await this.getDataFromMilvus(query, agent_id); const agent = await this.get_agent_details(agent_id); // console.log('agent', agent) this.setProvider(agent?.ai_model); const formattedContext = context.map((item: any, index: number) => { return `Document ${index + 1}:\n Score: ${item.score}\n Document ID: ${item.documentId}\n Metadata: ${item.metadata || 'N/A'}\n`; }).join('\n'); const contextString = ` Relevant Documents: ${formattedContext} `.trim(); const prompt = ` ${agent.call_prompt} Question: ${query} Context: ${contextString} this is our chat history answer accordingly ${chat_history} `.trim(); const params = { model: 'gpt-4o-mini', messages: [ { role: 'system', content: agent.call_prompt }, { role: 'user', content: query }, { role: 'system', content: contextString }, { role: 'assistant', content: chat_history }, ], max_tokens: 200, temperature: 0.7, }; let answer: any; console.time("AI RESPONSE TIME") console.log('provider ==========>> ', this.provider) if (this.provider === "chatgpt") { const response: OpenAI.Chat.ChatCompletion = await this.genAImodel.chat.completions.create(params); answer = response.choices[0].message.content; } else if (this.provider === "gemini") { const model = this.genAImodel.getGenerativeModel({ model: 'gemini-1.5-flash', generationConfig: { maxOutputTokens: 200, temperature: 0.7, }, }); const result = await model.generateContentStream(prompt); for await (const chunk of result.stream) { const chunkText = chunk.text(); // this.CallService.save_call_transcript({ // stream_id: stream_id, // text: data?.answer, // role: 'model', // }); ttsService.generate( { partialResponseIndex: null, partialResponse: chunkText, voice: agent.voice.toString() }, 0 ) } } console.timeEnd("AI RESPONSE TIME") // return { // answer, // }; } catch (error) { throw error; } } // async get_prompt(agent_id: string) { // try { // let data = await this.Model.AgentModel.findOne({ _id: new Types.ObjectId(agent_id) }, { call_prompt: 1 }, { lean: true }) // return data?.call_prompt // } catch (error) { // throw error // } // } async get_agent_details(agent_id: string) { try { let data = await this.Model.AgentModel.findOne({ _id: new Types.ObjectId(agent_id) }, { name: 1, call_prompt: 1, chat_prompt: 1, ai_model: 1, ai_model_version: 1, voice: 1 }, { lean: true }) return data } catch (error) { throw error } } private async create_new_chat(query: string, workspace_id: string, agent_id: string) { let chat_create = await this.Model.CallModel.create({ type: "VOICE_CHAT", last_message: query, workspace_id: new Types.ObjectId(workspace_id), agent_id: new Types.ObjectId(agent_id), created_at: +new Date() }) return chat_create._id.toString() } async getAnswerForLandingPage(body: dto.Knowledge.getLandingAnswer) { try { const { query, voice, secret_key, agent_id } = body; const find_workspace_key = await this.Model.workspaceKeysModel.findOne({ key: secret_key?.toLowerCase(), is_delete: false }) if (!find_workspace_key) throw new HttpException('scret key not found', HttpStatus.BAD_REQUEST) const subscription = await this.Model.subscriptionModel.findOne({ workspace_id: find_workspace_key.workspace_id, is_active: true }); if (!subscription) throw new HttpException(`You don’t have an active subscription plan. Please purchase a subscription to continue.`, HttpStatus.BAD_REQUEST); if (subscription.tele_duration_limit_left <= 0) { throw new HttpException(`You have reached your call limit hours. Please purchase additional credits or upgrade your plan to continue`, HttpStatus.BAD_REQUEST); } const chat_id = body.chat_id ? body.chat_id : await this.create_new_chat(query, find_workspace_key?.workspace_id, agent_id); await this.Model.CallModel.findByIdAndUpdate(new Types.ObjectId(chat_id), { last_message: query }); await this.save_chat({ role: "user", text: query, chat_id: chat_id }) const chat_history = await this.chat_history_for_chat(chat_id) const [context, agent] = await Promise.all([ this.getDataFromMilvus(query, agent_id), this.get_agent_details(agent_id), ]); this.setProvider(agent?.ai_model); console.log('MODEL: ==>', this.provider); const formattedContext = context.map((item: any, index: number) => ( `Document ${index + 1}:\n Score: ${item.score}\n Document ID: ${item.documentId}\n Metadata: ${item.metadata || 'N/A'}` )).join('\n'); const contextString = `Relevant Documents:\n${formattedContext}`.trim(); const prompt = ` ${agent.call_prompt} Question: ${query} Context: ${contextString} This is our chat history; answer accordingly: ${chat_history} `.trim(); const messages = [ { role: 'system', content: agent.call_prompt }, { role: 'user', content: query }, { role: 'system', content: contextString }, { role: 'assistant', content: chat_history }, ]; let response: { answer: string; token_uses: number }; switch (this.provider) { case "chatgpt": response = await this.getChatGPTAnswer(messages); break; case "gemini": response = await this.getGeminiAnswer(prompt); break; case "claud": response = await this.getClaudAnswer(prompt); break; case "deepseek": response = await this.getClaudAnswer(prompt); break; default: throw new Error(`Unsupported provider: ${this.provider}`); } const { answer, token_uses } = response; await this.save_chat({ role: "model", text: answer, chat_id: chat_id }) const data = await this.convert_tts(answer, agent.voice) return { data: data, chat_id, model_text: answer }; } catch (error) { throw error; } } private async delete_last_message_of_call(stream_id: any) { try { let data = await this.Model.CallTranscriptModel.findOne({ stream_id: stream_id, is_send: false, role: "user" }, {}, { lean: true, sort: { _id: -1 }, limit: 1 }) if (data?.role == "user") { return await this.Model.CallTranscriptModel.findOneAndUpdate({ _id: data?._id }, { is_delete: true }, { lean: true }) } } catch (error) { throw error } } private async get_last_message_of_call(chat_id: any) { try { let data = await this.Model.landingPageChatHistoryModel.findOne({ chat_id: chat_id, is_send: false, role: "user" }, {}, { lean: true, sort: { _id: -1 }, limit: 1 }) if (data) return data?.text return null } catch (error) { throw error } } async getAnswerForLandingPageV2(body: dto.Knowledge.getLandingAnswer, mode = "CALL") { try { let { query, voice, secret_key, chat_id } = body; const find_workspace_key = await this.Model.workspaceKeysModel.findOne({ key: secret_key?.toLowerCase(), is_delete: false }) if (!find_workspace_key) throw new HttpException('scret key not found', HttpStatus.BAD_REQUEST) const agent_id = await this.CallService.get_default_agent_id("WEB"); console.log('agent_id', agent_id) const [context, agent] = await Promise.all([ // this.getDataFromMilvus(query, agent_id), this.getDocumentLandingPage(query), this.get_agent_details(agent_id), ]); this.setProvider(agent?.ai_model); console.log('context, agent', context, agent) if (mode == "CALL") { const last_message = await this.get_last_message_of_call(chat_id); if (last_message) await this.delete_last_message_of_call(chat_id); query = last_message ? last_message + " " + query : query; } const save_chat_id = await this.save_chat({ role: "user", text: query, chat_id: chat_id }) console.log('save_chat_id', save_chat_id) const chat_history = await this.chat_history_for_chat(chat_id); console.log('chat_history', chat_history) const formattedContext = (context && context.length) ? context.map((item: any, index: number) => { return `Document ${index + 1}:\n Score: ${item.score}\n Document ID: ${item.documentId}\n Metadata: ${item.metadata || 'N/A'}\n`; }).join('\n') : ""; const contextString = `Relevant Documents:\n${formattedContext}`.trim(); const prompt = ` ${agent.call_prompt} Question: ${query} Context: ${contextString} This is our chat history; answer accordingly: ${chat_history} `.trim(); const messages = [ { role: 'system', content: agent.call_prompt }, { role: 'user', content: query }, { role: 'system', content: contextString }, { role: 'assistant', content: chat_history }, ]; const messages_claud = [ { role: 'system', content: 'You are a helpful AI assistant. Answer based on the given context and chat history.', }, { role: 'user', content: ` ${agent.call_prompt} Question: ${query} Context: ${contextString} This is our chat history; answer accordingly: ${chat_history} `.trim(), } ]; let response: { answer: string; token_uses: number }; console.time("AI RESPONSE TIME") console.log('provider ==========>> ', this.provider) switch (this.provider) { case "chatgpt": response = await this.getChatGPTAnswer(messages); break; case "gemini": response = await this.getGeminiAnswer(prompt); break; case "claud": response = await this.getClaudAnswer(prompt); break; case "deepseek": response = await this.getClaudAnswer(prompt); break; default: throw new Error(`Unsupported provider: ${this.provider}`); } const { answer, token_uses } = response; console.log('answer', answer) const reply = await this.save_chat({ role: "model", text: answer, chat_id: chat_id }) console.log('reply', reply) const query_update = await this.update_chat({ save_chat_id }) console.log('query_update', query_update) if (mode == "CHAT") { return { answer } } const data = await this.convert_tts(answer, agent.voice) return { data: data }; } catch (error) { throw error; } } async chat_history_for_chat(chat_id: string) { try { // let call = await this.model.CallModel.findOne({ _id: chat_id }) let chatData = await this.Model.landingPageChatHistoryModel.find({ chat_id: chat_id }, {}, { sort: { _id: 1 } }) let chat = chatData .map((message) => `${message.role.charAt(0).toUpperCase() + message.role.slice(1)}: ${message.text}`) .join('\n'); return chat } catch (error) { throw error } } async save_chat(data: any) { try { let { role, text, chat_id } = data const result = await this.Model.landingPageChatHistoryModel.create({ text: text, role: role, chat_id: chat_id, created_at: + new Date() }) return result._id } catch (error) { throw error } } async update_chat(data: any) { try { const { save_chat_id } = data await this.Model.landingPageChatHistoryModel.findOneAndUpdate( { _id: save_chat_id }, { is_send: true }, { new: true }, ); } catch (error) { throw error } } async convert_tts(text: any, voice: string) { try { const response = await axios.post( `https://api.deepgram.com/v1/speak?model=${voice ? voice : "aura-asteria-en"}`, { text: text }, { headers: { 'Content-Type': 'application/json', Authorization: `Token ${process.env.DEEPGRAM_API_KEY}`, }, responseType: 'arraybuffer', // To get the response as a buffer } ); if (response.status === 200) { return response.data; } return null } catch (error) { throw error } } async knowledgeBaseUploadLandingPage(url: string) { try { const { data: html } = await axios.get(url); const $ = cheerio.load(html); const textContent = $('body').text().trim(); let embeddings = this.getChunks(textContent); let data_to_push = []; for (let i = 0; i < embeddings.length; i++) { const content = embeddings[i]; let embedding = this.addDataToMilvusLandingPage(content); data_to_push.push(embedding); if (data_to_push.length === 10) { let resolve = await Promise.all(data_to_push); data_to_push = []; } } await Promise.all(data_to_push); return { message: "success" } } catch (error) { throw error; } } async addDataToPineconeLandingPage(content: any) { try { content = content.trim().replace(/\s+/g, ''); const embedding = await this.embedContent(content); let id = `docs ${+new Date()}` let vector = [ { id: id, values: embedding, metadata: { text: content, knowledge_base_id: "landing_page" }, }, ]; this.addDocument(vector); return embedding; } catch (error) { throw error; } } async addDataToMilvusLandingPage(content: any) { try { content = content.trim().replace(/\s+/g, ''); const embedding = await this.embedContent(content); let id = `docs ${+new Date()}` let vector = [ { id: id, values: embedding, metadata: { text: content, knowledge_base_id: "landing_page" }, }, ]; this.addDocument(vector); return embedding; } catch (error) { throw error; } } async getDocumentLandingPage(query: string) { try { console.time('Embedding generation'); const vectorQuery = await this.embedContent(query); console.timeEnd('Embedding generation'); const web_agent = await this.Model.AgentModel.findOne({ is_default: true, default_type: "WEB" }, { knowledge_base_id_for_website: 1 }, { lean: true }); console.log('web_agent', web_agent) const documents = await this.Model.documentsModel.find({ knowledge_base_id: new Types.ObjectId(String(web_agent.knowledge_base_id_for_website)), }, { _id: 1 }, { lean: true }); const document_ids = documents.map((elem) => { return elem?._id.toString() }); const expr = `documentId in [${document_ids.map(id => `'${id}'`).join(', ')}]`; await this.loadCollectionIfNeeded(); const searchParams = { collection_name: this.collectionName, vectors: vectorQuery, search_params: { anns_field: 'values', topk: 3, metric_type: 'L2', params: JSON.stringify({ nprobe: 10 }), }, expr: expr, output_fields: ['id', 'metadata', 'documentId'], }; console.time('Milvus Vector DB'); const searchResponse = await this.milvusClient.search(searchParams); console.timeEnd('Milvus Vector DB'); if (searchResponse.status.error_code !== 'Success') { throw new Error(`Search failed: ${searchResponse.status.reason}`); } return searchResponse.results.map((result) => ({ score: result.score, documentId: result.documentId, metadata: result.metadata, })); } catch (error) { throw error; } } async getEmotion(chat_history?: string) { try { const prompt = ` You are an AI model analyzing customer interactions. Below is a chat history between a customer and a support agent. Your task is to determine the overall customer sentiment at the end of the conversation. Use labels like "Customer Satisfied," "Customer Frustrated," "Customer Angry," "Customer Interested," "Customer uninterested," or "Customer Indifferent" to summarize the sentiment. Be concise and provide only the sentiment label. Chat History: ${chat_history} What is the overall customer sentiment?`.trim(); const model = this.genAI.getGenerativeModel({ model: 'gemini-1.5-flash', generationConfig: { // candidateCount: 1, // stopSequences: ["x"], maxOutputTokens: 200, temperature: 0.7, }, }); console.log('getEmotion(chat_history?: string)') console.time('time in gemini response'); const result = await model.generateContent(prompt); console.timeEnd('time in gemini response'); return { answer: result.response.text(), }; } catch (error) { throw error; } } async getChatStatus(chat_history?: string) { try { const prompt = ` You are an AI model analyzing customer interactions. Below is a chat history between a customer and a support agent. Your task is to determine the overall customer chat interaction. Use labels like "successful," "disrupted," "unsucessful" to summarize the chat status. Be concise and provide only the label i give you. Chat History: ${chat_history} What is the overall customer chat status?`.trim(); const model = this.genAI.getGenerativeModel({ model: 'gemini-1.5-flash', generationConfig: { // candidateCount: 1, // stopSequences: ["x"], maxOutputTokens: 200, temperature: 0.7, }, }); console.log('getChatStatus(chat_history?: string)') console.time('time in gemini response'); const result = await model.generateContent(prompt); console.timeEnd('time in gemini response'); return { answer: result.response.text(), }; } catch (error) { throw error; } } async getCustomerName(chat_history?: string) { try { const prompt = ` You are an AI model analyzing customer interactions. Below is a chat history between a customer and a support agent: Chat History: ${chat_history} Your task is to identify the name of the customer from the chat history. If a name is identified, respond in the format: Customer Name: [Name] If the name is not available, respond with: Customer Name: N/A.` const model = this.genAI.getGenerativeModel({ model: 'gemini-1.5-flash', generationConfig: { // candidateCount: 1, // stopSequences: ["x"], maxOutputTokens: 200, temperature: 0.7, }, }); console.log("getCustomerName") console.time('time in gemini response'); const result = await model.generateContent(prompt); console.timeEnd('time in gemini response'); return { answer: result.response.text(), }; } catch (error) { throw error; } } async getExtractedData(chat_history?: string) { try { const prompt = ` You are an AI model analyzing customer interactions. Below is a chat history between a customer and a support agent. Your task is to extract key information from the conversation and format it as JSON. The required fields are: - full_name: Full name of the person (if mentioned). - email: Email address (if provided). - phone_number: Contact number (if provided). - address: Residential or business address (if available). - purpose_of_appointment: The reason for the appointment or meeting. - appointment_date_time: Date and time of the appointment or meeting. - meeting_with: Name of the person the customer has an appointment with. - meeting_details: Additional information related to the meeting, if any. If any information is not available in the chat, set its value to null. Chat History: ${chat_history} Provide the extracted information in the following JSON format: { "full_name": "John Doe", "email": "johndoe@example.com", "phone_number": "+123456789", "address": "123 Main St, City, Country", "purpose_of_appointment": "Consultation regarding project", "appointment_date_time": "2024-06-15T14:30:00", "meeting_with": "Jane Smith", "meeting_details": "Discussion about project requirements" } If data is missing, use null. Provide only the JSON output without additional explanations. `.trim(); const model = this.genAI.getGenerativeModel({ model: 'gemini-1.5-flash', generationConfig: { maxOutputTokens: 200, temperature: 0.7, }, }); console.time('time in gemini response'); const result = await model.generateContent(prompt); console.timeEnd('time in gemini response'); return { answer: result.response.text(), }; } catch (error) { throw error; } } async createCollectionIfNotExists() { try { const collections = await this.milvusClient.showCollections(); console.log('milvus collections', collections.collections) const dim = 728; const field_name = [ { name: 'id', data_type: DataType.VarChar, is_primary_key: true, max_length: 50, }, { name: 'metadata', data_type: DataType.VarChar, max_length: 10000, }, { name: 'values', data_type: DataType.FloatVector, type_params: { dim: `${dim}` }, }, { name: 'documentId', data_type: DataType.VarChar, max_length: 1000, }, ] if (!collections.collection_names.includes(this.collectionName)) { console.log(`Creating collection: ${this.collectionName}`); const test = await this.milvusClient.createCollection({ collection_name: this.collectionName, fields: field_name, }); console.log('JSON.stringify(test', JSON.stringify(test)) console.log(`Collection created: ${this.collectionName}`); await this.checkAndCreateIndexMilvus(); return true; } } catch (error) { throw error; } } async checkAndCreateIndexMilvus() { try { const indexParams = { collection_name: this.collectionName, field_name: 'values', index_type: 'IVF_FLAT', metric_type: 'L2', params: { nlist: 128 }, }; const response = await this.milvusClient.createIndex(indexParams); return response } catch (error) { console.error(`Error checking or creating index: ${error.message}`); throw error; } } async loadCollectionIfNeeded() { try { const collections = await this.milvusClient.showCollections(); const collection = collections.data.find((col) => col.name === this.collectionName); if (!collection) { throw new Error(`Collection '${this.collectionName}' does not exist.`); } if (!collection.loaded) { console.log(`Collection '${this.collectionName}' not loaded. Loading it now...`); await this.milvusClient.loadCollection({ collection_name: this.collectionName }); console.log(`Collection '${this.collectionName}' loaded successfully.`); } return true; } catch (error) { throw error; } } async getDataFromMilvus(queryVector: string, agent_id: string) { try { console.time('Embedding generation'); const vectorQuery = await this.embedContent(queryVector); console.timeEnd('Embedding generation'); const document_ids = await this.getKnowledgeBaseDocuments(agent_id); const expr = `documentId in [${document_ids.map(id => `'${id}'`).join(', ')}]`; console.log('expr ===========> docIds', document_ids.length) await this.loadCollectionIfNeeded(); const searchParams = { collection_name: this.collectionName, vectors: vectorQuery, search_params: { anns_field: 'values', topk: 3, metric_type: 'L2', params: JSON.stringify({ nprobe: 10 }), }, expr: expr, output_fields: ['id', 'metadata', 'documentId'], }; console.time('searchResponse vector'); const searchResponse = await this.milvusClient.search(searchParams); console.timeEnd('searchResponse vector'); return searchResponse.results.map((result) => ({ score: result.score, documentId: result.documentId, metadata: result.metadata, })); } catch (error) { console.error(`Error querying Milvus: ${error.message}`, { stack: error.stack, }); throw error; } } }