import dotenv from "dotenv";  
dotenv.config();  
const sbApiKey = process.env.SUPABASE\_API\_KEY;  
const sbUrl = process.env.SUPABASE\_URL;  
const sbClient = createClient(sbUrl, sbApiKey, {  
    auth: {  
        autoRefreshToken: false,  
        persistSession: false,  
        detectSessionInUrl: false  
    }  
});  

async function authenticate() {  
    const { user, error } = await sbClient.auth.signInWithPassword({  
        email: "fill actual email",  
        password: "fill with password",  
    });  

    if (error) {  
        console.log("error", error);  
    } else {  

    }  
}  

export function addDocumentData(contents) {  
    return new Promise((resolve, reject) => {  
        authenticate()  
            .then(() => {  
                const documents = contents.map(content => ({ content }));  

                sbClient  
                    .from("documentRaw")  
                    .insert(documents)  
                    .select()  
                    .then((response) => {  
                        const { data, error } = response;  
                        if (error) {  
                            console.log(error);  
                            reject(error);  
                        } else {  
                            resolve(data);  
                        }  
                    })  
                    .catch((error) => {  
                        console.log(error);  
                        reject(error);  
                    });  
            })  
            .catch((error) => {  
                console.log(error);  
                reject(error);  
            });  
    });  
}  
export function getDocumentData(ids) {  
    return new Promise((resolve, reject) => {  
        authenticate()  
            .then(() => {  
                const query = sbClient.from("documentRaw").select("\*");  
                if (Array.isArray(ids)) {  
                    query.in("id", ids);  
                } else {  
                    query.eq("id", ids);  
                }  

                query  
                    .then((response) => {  
                        const { data, error } = response;  
                        if (error) {  
                            console.log(error);  
                            reject(error);  
                        } else {  
                            if (Array.isArray(ids)) {  
                                resolve(data);  
                            } else {  
                                resolve(data\[0\]);  
                            }  
                        }  
                    })  
                    .catch((error) => {  
                        console.log(error);  
                        reject(error);  
                    });  
            })  
            .catch((error) => {  
                console.log(error);  
                reject(error);  
            });  
    });  
}

import { TextLoader } from "langchain/document\_loaders/fs/text";  
import { DirectoryLoader } from "langchain/document\_loaders/fs/directory";  
import dotenv from "dotenv";  
dotenv.config();  
import { OpenAIEmbeddings } from "@langchain/openai";  
import { CharacterTextSplitter } from "langchain/text\_splitter";  
import { SupabaseVectorStore } from "@langchain/community/vectorstores/supabase"  
import { createClient } from "@supabase/supabase-js";  
import { Document } from "langchain/document";  


const sbApiKey = process.env.SUPABASE\_API\_KEY;  
const sbUrl = process.env.SUPABASE\_URL;  
const client = createClient(sbUrl, sbApiKey); // initializing SupabaseClient  

const embeddings = new OpenAIEmbeddings({ // Embedding Model  
    model: "text-embedding-3-large",  
});  

const path = "./documents/";  
const directoryLoader = new DirectoryLoader(path, { // Loading multiple txt files.  
    ".txt": (path) => new TextLoader(path),  
});  
const splitter = new CharacterTextSplitter({ // Splitting text into chunks.  
    keepSeparator: false,  
    chunkSize: 1500,  
});  
import pit from "p-iteration"   // npm install p-iteration  
import { default as summarizer } from "./summarizeChain.js"; // summarizing content with anthropic, see below code  
import { addDocumentData } from "./supabaseDocument.js"; // we created this earlier.  
(async () => {  
    const loaded\_docs = await directoryLoader.load(); // load all documents  
    const text = await splitter.splitDocuments(loaded\_docs); // split documents into chunks. This returns an array of Document(s)  
    const textSummarized = await pit.mapSeries(text, async document => { //we are mapping each document to process it to another documents array.  
        let addToSupabaseRaw = await addDocumentData(\[document.pageContent\]) // add pageContent to documentsRaw Table in Supabase  
        const summary = await summarizer(document.pageContent); // summarize content and extract keywords and keyword clouds.  
        console.log("created row,", addToSupabaseRaw\[0\].id, summary.content);   
        return new Document({  
            metadata: { ...document.metadata, rawDocumentId: addToSupabaseRaw\[0\].id },  
            pageContent: summary  
        }); // this is our new Document, now pagecontent is summarized and we have rawDocumentId metadata to create relation later.  

    });  
    const vectorStore = await SupabaseVectorStore.fromDocuments(  
        textSummarized,  
        embeddings,  
        {  
            client,  
        }  
    );  // this step creates embeddings and saves them in SupabaseVector Store.  
})();

import dotenv from "dotenv";  
dotenv.config();  

import { PromptTemplate } from "@langchain/core/prompts";  
import { ChatAnthropic } from "@langchain/anthropic";  
import { StringOutputParser } from "@langchain/core/output\_parsers";  


let anthropicModel = "claude-3-haiku-20240307"  
const model = new ChatAnthropic({  
  temperature: 0.9,  
  model: anthropicModel,  
  // In Node.js defaults to process.env.ANTHROPIC\_API\_KEY,  
  // apiKey: "YOUR-API-KEY",  
  maxTokens: 4096,  
}); //setup LLM  

const summarizeInputTemplate = \`Without including personal information and private details, convert the given text into a tag cloud.  
In this context, embeddings will be created so that they can be easily retrieved when another text is used.  
Therefore, summarize what the text is about, what its demands are, and what it pertains to, generating keywords.  
Create very detailed groups of keywords.  
There's no need for proper names or place names. Look at the big picture; you're an excellent content analyst.  
Here's the text: {content} \`; // This is our template we can invoke it and change {content}   
const summarizeInputPrompt = PromptTemplate.fromTemplate(  
  summarizeInputTemplate  
);  
const summarizeChain = summarizeInputPrompt  
  .pipe(model).pipe(new StringOutputParser());  

const run = async (content) => { //this is our main function which receives a content and returns summarized content.  
  const response = await summarizeChain.invoke({  
    content: content,  
  });  
  return response  

};  

export default run;

import { SupabaseVectorStore } from "@langchain/community/vectorstores/supabase";  
import { createClient } from "@supabase/supabase-js";  
import { OpenAIEmbeddings } from "@langchain/openai";  

import dotenv from "dotenv";  
dotenv.config();  


const sbApiKey = process.env.SUPABASE\_API\_KEY;  
const sbUrl = process.env.SUPABASE\_URL;  
const client = createClient(sbUrl, sbApiKey);  


const embeddings = new OpenAIEmbeddings({  
    model: "text-embedding-3-large",  
});  
import pit from "p-iteration"  //npm install p-iteration  
import { getDocumentData } from "./supabaseDocument.js"; // we created this earlier, now we are using getDocumentData function.  
import { Document } from "langchain/document";  

const run = async () => {  
    const vectorStore = new SupabaseVectorStore( //initializing vector store.  
        embeddings,  
        {  
            client,  
            tableName: "documents",  
            queryName: "match\_documents",  
        }  
    );  
    let query = "your query for searching"  
//In here, we can create another chain similar to summarizer, but for analyzing query and creating keyword cloud out of it for further improvements.  
    const results = await vectorStore.similaritySearch(query, 5); // Now we searched in summarized and embeddings ( Keyword clouds).  
    const resultsRestored = await pit.mapSeries(results, async document => {  
        let getFromSupabase = await getDocumentData(document.metadata.rawDocumentId); // getting rawDocumentID from metadata and getting actual content from supabase table ( documentsRaw)  
        return new Document({  
            metadata: { ...document.metadata },  
            pageContent: getFromSupabase.content  
        }); //populating new documents with actual data  
    });  
   console.log(resultsRestored);  

};  
run()