RAG 聊天机器人：用 Langchain 和 Streamlit开启与 PDF 的智能对话

import streamlit as st  # Importing Streamlit for the web interfacefrom PyPDF2 import PdfReader  # Importing PyPDF2 for reading PDF filesfrom langchain.text_splitter import RecursiveCharacterTextSplitter  # Importing Langchain's text splitterfrom langchain_core.prompts import ChatPromptTemplate  # Importing ChatPromptTemplate from Langchainfrom langchain_community.embeddings.spacy_embeddings import SpacyEmbeddings  # Importing SpacyEmbeddingsfrom langchain_community.vectorstores import FAISS  # Importing FAISS for vector storefrom langchain.tools.retriever import create_retriever_tool  # Importing retriever tool from Langchainfrom dotenv import load_dotenv  # Importing dotenv to manage environment variablesfrom langchain_anthropic import ChatAnthropic  # Importing ChatAnthropic from Langchainfrom langchain_openai import ChatOpenAI, OpenAIEmbeddings  # Importing ChatOpenAI and OpenAIEmbeddings from Langchainfrom langchain.agents import AgentExecutor, create_tool_calling_agent  # Importing agent-related modules from Langchain
import osos.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"  # Setting environment variable to avoid potential conflicts

def pdf_read(pdf_doc):    text = ""    for pdf in pdf_doc:        pdf_reader = PdfReader(pdf)  # Initializing the PDF reader for the given document        for page in pdf_reader.pages:  # Iterating through each page in the PDF            text += page.extract_text()  # Extracting and appending the text from the current page    return text  # Returning the concatenated text from all pages

def get_chunks(text):    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)  # Initializing the text splitter with specified chunk size and overlap    chunks = text_splitter.split_text(text)  # Splitting the text into chunks    return chunks  # Returning the list of text chunks

embeddings = SpacyEmbeddings(model_name="en_core_web_sm")  # Initializing Spacy embeddings with the specified modeldef vector_store(text_chunks):    vector_store = FAISS.from_texts(text_chunks, embedding=embeddings)  # Creating a FAISS vector store from text chunks    vector_store.save_local("faiss_db")  # Saving the vector store locally

def get_conversational_chain(tools, ques):    # Initialize the language model with specified parameters    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0, api_key="")        # Define the prompt template for guiding the AI's responses    prompt = ChatPromptTemplate.from_messages(        [            (                "system",                """You are a helpful assistant. Answer the question as detailed as possible from the provided context, make sure to provide all the details. If the answer is not in                provided context just say, "answer is not available in the context", don't provide the wrong answer""",            ),            ("placeholder", "{chat_history}"),            ("human", "{input}"),            ("placeholder", "{agent_scratchpad}"),        ]    )        # Create the tool list and the agent    tool = [tools]    agent = create_tool_calling_agent(llm, tool, prompt)        # Execute the agent to process the user's query and get a response    agent_executor = AgentExecutor(agent=agent, tools=tool, verbose=True)    response = agent_executor.invoke({"input": ques})    print(response)    st.write("Reply: ", response['output'])

def user_input(user_question):    # Load the vector database    new_db = FAISS.load_local("faiss_db", embeddings, allow_dangerous_deserialization=True)        # Create a retriever from the vector database    retriever = new_db.as_retriever()    retrieval_chain = create_retriever_tool(retriever, "pdf_extractor", "This tool is to give answers to queries from the PDF")        # Get the conversational chain to generate a response    get_conversational_chain(retrieval_chain, user_question)

def main():    st.set_page_config("Chat PDF")    st.header("RAG based Chat with PDF")user_question = st.text_input("Ask a Question from the PDF Files")    if user_question:        user_input(user_question)    with st.sidebar:        st.title("Menu:")        pdf_doc = st.file_uploader("Upload your PDF Files and Click on the Submit & Process Button", accept_multiple_files=True)        if st.button("Submit & Process"):            with st.spinner("Processing..."):                raw_text = pdf_read(pdf_doc)                text_chunks = get_chunks(raw_text)                vector_store(text_chunks)                st.success("Done")if __name__ == "__main__":    main()

import streamlit as stfrom PyPDF2 import PdfReaderfrom langchain.text_splitter import RecursiveCharacterTextSplitterfrom langchain_core.prompts import ChatPromptTemplatefrom langchain_community.embeddings.spacy_embeddings import SpacyEmbeddingsfrom langchain_community.vectorstores import FAISSfrom langchain.tools.retriever import create_retriever_toolfrom dotenv import load_dotenvfrom langchain_anthropic import ChatAnthropicfrom langchain_openai import ChatOpenAI, OpenAIEmbeddingsfrom langchain.agents import AgentExecutor, create_tool_calling_agent
import osos.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE"embeddings = SpacyEmbeddings(model_name="en_core_web_sm")def pdf_read(pdf_doc):    text = ""    for pdf in pdf_doc:        pdf_reader = PdfReader(pdf)        for page in pdf_reader.pages:            text += page.extract_text()    return textdef get_chunks(text):    text_splitter = RecursiveCharacterTextSplitter(chunk_size=1000, chunk_overlap=200)    chunks = text_splitter.split_text(text)    return chunksdef vector_store(text_chunks):    vector_store = FAISS.from_texts(text_chunks, embedding=embeddings)    vector_store.save_local("faiss_db")def get_conversational_chain(tools, ques):    llm = ChatOpenAI(model_name="gpt-3.5-turbo", temperature=0, api_key="")    prompt = ChatPromptTemplate.from_messages(        [            (                "system",                """You are a helpful assistant. Answer the question as detailed as possible from the provided context, make sure to provide all the details. If the answer is not in                provided context just say, "answer is not available in the context", don't provide the wrong answer""",            ),            ("placeholder", "{chat_history}"),            ("human", "{input}"),            ("placeholder", "{agent_scratchpad}"),        ]    )    tool = [tools]    agent = create_tool_calling_agent(llm, tool, prompt)    agent_executor = AgentExecutor(agent=agent, tools=tool, verbose=True)    response = agent_executor.invoke({"input": ques})    print(response)    st.write("Reply: ", response['output'])def user_input(user_question):    new_db = FAISS.load_local("faiss_db", embeddings, allow_dangerous_deserialization=True)    retriever = new_db.as_retriever()    retrieval_chain = create_retriever_tool(retriever, "pdf_extractor", "This tool is to give answers to queries from the PDF")    get_conversational_chain(retrieval_chain, user_question)def main():    st.set_page_config("Chat PDF")    st.header("RAG based Chat with PDF")    user_question = st.text_input("Ask a Question from the PDF Files")    if user_question:        user_input(user_question)    with st.sidebar:        st.title("Menu:")        pdf_doc = st.file_uploader("Upload your PDF Files and Click on the Submit & Process Button", accept_multiple_files=True)        if st.button("Submit & Process"):            with st.spinner("Processing..."):                raw_text = pdf_read(pdf_doc)                text_chunks = get_chunks(raw_text)                vector_store(text_chunks)                st.success("Done")if __name__ == "__main__":    main()

streamlit run app.py