{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from unstructured.partition.pdf import partition_pdf\n",
    "\n",
    "output_path = \"/home/sepehr/dev/rag/document/\"\n",
    "file_path = \"/home/sepehr/dev/rag/document/NIPS-2017-attention-is-all-you-need-Paper.pdf\"\n",
    "\n",
    "# Reference: https://docs.unstructured.io/open-source/core-functionality/chunking\n",
    "chunks = partition_pdf(\n",
    "    filename=file_path,\n",
    "    infer_table_structure=True,            # extract tables\n",
    "    strategy=\"hi_res\",                     # mandatory to infer tables\n",
    "\n",
    "    extract_image_block_types=[\"Image\"],   # Add 'Table' to list to extract image of tables\n",
    "    # image_output_dir_path=output_path,   # if None, images and tables will saved in base64\n",
    "\n",
    "    extract_image_block_to_payload=True,   # if true, will extract base64 for API usage\n",
    "\n",
    "    chunking_strategy=\"by_title\",          # or 'basic'\n",
    "    max_characters=10000,                  # defaults to 500\n",
    "    combine_text_under_n_chars=2000,       # defaults to 0\n",
    "    new_after_n_chars=6000,\n",
    "\n",
    "    # extract_images_in_pdf=True,          # deprecated\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{\"<class 'unstructured.documents.elements.CompositeElement'>\"}"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "set([str(type(el)) for el in chunks])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': 'Image',\n",
       " 'element_id': '7035e471-b3af-4b84-9cf3-23a1a493dbe2',\n",
       " 'text': '',\n",
       " 'metadata': {'coordinates': {'points': ((486.0, 261.1805555555558),\n",
       "    (486.0, 614.7805555555556),\n",
       "    (664.0, 614.7805555555556),\n",
       "    (664.0, 261.1805555555558)),\n",
       "   'system': 'PixelSpace',\n",
       "   'layout_width': 1700,\n",
       "   'layout_height': 2200},\n",
       "  'last_modified': '2025-03-01T19:37:35',\n",
       "  'filetype': 'PPM',\n",
       "  'languages': ['eng'],\n",
       "  'page_number': 4,\n",
       "  'image_base64': '[encoded data, 16772 characters]',\n",
       "  'image_mime_type': 'image/jpeg'}}"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# This is what an extracted image looks like.\n",
    "# It contains the base64 representation only because we set the param extract_image_block_to_payload=True\n",
    "\n",
    "elements = chunks[3].metadata.orig_elements\n",
    "chunk_images = [el for el in elements if 'Image' in str(type(el))]\n",
    "chunk_images[0].to_dict()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'type': 'Image',\n",
       " 'element_id': '7035e471-b3af-4b84-9cf3-23a1a493dbe2',\n",
       " 'text': '',\n",
       " 'metadata': {'coordinates': {'points': ((486.0, 261.1805555555558),\n",
       "    (486.0, 614.7805555555556),\n",
       "    (664.0, 614.7805555555556),\n",
       "    (664.0, 261.1805555555558)),\n",
       "   'system': 'PixelSpace',\n",
       "   'layout_width': 1700,\n",
       "   'layout_height': 2200},\n",
       "  'last_modified': '2025-03-01T19:37:35',\n",
       "  'filetype': 'PPM',\n",
       "  'languages': ['eng'],\n",
       "  'page_number': 4,\n",
       "  'image_base64': '[encoded data, 16772 characters]',\n",
       "  'image_mime_type': 'image/jpeg'}}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "chunk_images = [el for el in elements if 'Image' in str(type(el))]\n",
    "chunk_images[0].to_dict()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "# separate tables from texts\n",
    "tables = []\n",
    "texts = []\n",
    "\n",
    "for chunk in chunks:\n",
    "    if \"Table\" in str(type(chunk)):\n",
    "        tables.append(chunk)\n",
    "\n",
    "    if \"CompositeElement\" in str(type((chunk))):\n",
    "        texts.append(chunk)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Get the images from the CompositeElement objects\n",
    "def get_images_base64(chunks):\n",
    "    images_b64 = []\n",
    "    for chunk in chunks:\n",
    "        if \"CompositeElement\" in str(type(chunk)):\n",
    "            chunk_els = chunk.metadata.orig_elements\n",
    "            for el in chunk_els:\n",
    "                if \"Image\" in str(type(el)):\n",
    "                    images_b64.append(el.metadata.image_base64)\n",
    "    return images_b64\n",
    "\n",
    "images = get_images_base64(chunks)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/jpeg": "[encoded data, 27564 characters]",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import base64\n",
    "from IPython.display import Image, display\n",
    "\n",
    "def display_base64_image(base64_code):\n",
    "    # Decode the base64 string to binary\n",
    "    image_data = base64.b64decode(base64_code)\n",
    "    # Display the image\n",
    "    display(Image(data=image_data))\n",
    "\n",
    "display_base64_image(images[2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.prompts import ChatPromptTemplate\n",
    "from langchain_ollama.llms import OllamaLLM\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "# Prompt\n",
    "prompt_text = \"\"\"\n",
    "You are an assistant tasked with summarizing tables and text.\n",
    "Give a concise summary of the table or text.\n",
    "\n",
    "Respond only with the summary, no additionnal comment.\n",
    "Do not start your message by saying \"Here is a summary\" or anything like that.\n",
    "Just give the summary as it is.\n",
    "\n",
    "Table or text chunk: {element}\n",
    "\n",
    "\"\"\"\n",
    "prompt = ChatPromptTemplate.from_template(prompt_text)\n",
    "\n",
    "model = OllamaLLM(base_url=\"172.20.48.1:11434\",\n",
    "    model=\"llama3.2\")\n",
    "summarize_chain = {\"element\": lambda x: x} | prompt | model | StrOutputParser()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "text_summaries = summarize_chain.batch(texts, {\"max_concurrency\": 3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Researchers at Google propose the Transformer, a simple neural network architecture that relies solely on attention mechanisms to draw global dependencies between input and output. The model achieves state-of-the-art results in machine translation tasks, outperforming recurrent models with significant computational efficiency gains.',\n",
       " 'Background: \\nReducing sequential computation forms the foundation of Extended Neural GPU, ByteNet, ConvS2S, which use convolutional neural networks as building blocks. \\nSelf-attention is an attention mechanism relating different positions within a single sequence to compute representations. \\nEnd-to-end memory networks rely on recurrent attention mechanisms and perform well in language-related tasks.\\nThe Transformer is the first transduction model relying entirely on self-attention to compute input and output representations without using RNNs or convolution.',\n",
       " 'The Transformer model consists of two identical stacks of 6 layers: an encoder stack and a decoder stack. Each layer has three sub-layers: multi-head self-attention, feed-forward network, and residual connections followed by layer normalization. The decoder adds a third sub-layer for multi-head attention over the output of the encoder stack, with masking to prevent positions from attending to subsequent positions.',\n",
       " 'Scaled Dot-Product Attention computes the dot products of queries with keys, divided by dim_k, and applies softmax to get weights on values. This is done on a set of queries simultaneously, packed into a matrix Q, with keys and values also packed into matrices K and V. The output matrix is calculated as: Attention(Q,K,V) = softmax(Q*K*sqrt(dk)) * V.\\n\\nMulti-Head Attention combines multiple attention layers (heads) in parallel, where each head projects the queries, keys, and values to different dimensions and applies the dot-product attention.',\n",
       " 'The Transformer uses multi-head attention in three ways:\\n\\n• Encoder-decoder attention: queries from decoder, memory keys and values from encoder\\n• Self-attention: all positions attend over all previous positions\\n• Decoder self-attention: each position attends to all previous positions (but with leftward information flow masking)\\n\\nEach layer contains a fully connected feed-forward network applied separately and identically. Learned embeddings convert input and output tokens to vectors, and a shared weight matrix is used for the linear transformation and softmax function.\\n\\nFFN(x) = max(0,xW1 + b1)W2 + b2\\n\\nEmbeddings and softmax are used to convert tokens to vectors and predict next-token probabilities respectively.',\n",
       " 'Positional encoding is added to input embeddings to capture sequence order due to lack of recurrence or convolution. Sine and cosine functions are used with frequencies that form a geometric progression from 2π to 10000·2π for positional encodings, with similar results obtained from learned embeddings.',\n",
       " 'Training regime for models: trained on standard WMT 2014 datasets using Adam optimizer with varying learning rates and regularization techniques including Residual Dropout and label smoothing; base models took 12 hours, while big models took 3.5 days to train.',\n",
       " \"The transformer model achieved state-of-the-art BLEU scores on English-to-German and English-to-French translation tasks, outperforming previously published models by a significant margin. Training costs were reduced by using a smaller model size.\\n\\nTable 3 shows the effect of varying attention heads, key and value dimensions, and other components on the transformer model's performance, which varied in BLEU scores but generally showed improvements over the base model.\\n\\nVariations in model architecture resulted in changes to training time, computational cost, and perplexity metrics.\",\n",
       " \"It seems you have provided a large text file containing a list of papers related to deep learning and natural language processing. The text appears to be in a format that is not easily readable or understandable.\\n\\nIf you could provide more context or clarify what specific information or insights you would like me to extract from this text, I'll do my best to help. Alternatively, if you'd like me to summarize the content of the papers listed, I can try to provide a brief summary for each paper. Please let me know how I can assist you further.\"]"
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "text_summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Summarize tables\n",
    "tables_html = [table.metadata.text_as_html for table in tables]\n",
    "table_summaries = summarize_chain.batch(tables_html, {\"max_concurrency\": 3})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tables_html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# import getpass\n",
    "# import os\n",
    "\n",
    "# if not os.getenv(\"DEEPSEEK_API_KEY\"):\n",
    "#     os.environ[\"DEEPSEEK_API_KEY\"] = getpass.getpass(\"Enter your DeepSeek API key: \")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_deepseek import ChatDeepSeek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"OPENAI_API_KEY\"] = \"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\"\n",
    "from langchain_openai import ChatOpenAI\n",
    "prompt_template = \"\"\"Describe the image in detail. For context,\n",
    "                  the image is part of a outlines the methodology for factorial complete plans of experimentss.\"\"\"\n",
    "messages = [\n",
    "    (\n",
    "        \"user\",\n",
    "        [\n",
    "            {\"type\": \"text\", \"text\": prompt_template},\n",
    "            {\n",
    "                \"type\": \"image_url\",\n",
    "                \"image_url\": {\"url\": \"data:image/jpeg;base64,{image}\"},\n",
    "            },\n",
    "        ],\n",
    "    )\n",
    "]\n",
    "\n",
    "prompt = ChatPromptTemplate.from_messages(messages)\n",
    "\n",
    "# chain = OllamaLLM(base_url=\"172.20.48.1:11434\",\n",
    "#      model=\"minicpm-v\")\n",
    "chain = prompt | ChatOpenAI(model=\"gpt-4o-mini\") | StrOutputParser()\n",
    "\n",
    "image_summaries = chain.batch(images)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['The image depicts a detailed diagram of a neural network architecture, specifically in the context of transformer models used for processing sequences, such as in natural language processing.\\n\\n1. **Structure**: The diagram is divided into two main sections. The left side shows the encoding layer, while the right side illustrates the decoding layer.\\n\\n2. **Layers**:\\n   - **Input Embedding**: At the bottom, the input embeddings are represented, which are transformed into a higher-dimensional space suitable for input into the network.\\n   - **Positional Encoding**: Positioned near the input layer, this component provides information about the order of the sequence, as traditional neural networks lack an understanding of position.\\n   \\n3. **Encoding Block**: On the left:\\n   - Several blocks labeled \"Multi-Head Attention\" and \"Feed Forward\" are stacked vertically. \\n   - Each block is connected with arrows indicating the flow of data.\\n   - The blocks are enclosed in boxes labeled \"Add & Norm,\" which likely represents operations for residual connections and normalization processes that stabilize and improve the training process.\\n\\n4. **Decoding Block**: On the right:\\n   - Similar to the left side, but includes blocks labeled \"Masked Multi-Head Attention.\" This indicates that during training, the model can only attend to previous positions in the sequence to predict the next output, preventing future information from influencing past decisions.\\n   - There is also a \"Feed Forward\" layer followed by \"Add & Norm\" sections, similar to those on the encoding side.\\n\\n5. **Output Layer**: At the top of the right section, outputs are generated through a \"Linear\" layer followed by a \"Softmax\" function that produces probabilities for the output tokens.\\n\\n6. **N x**: This notation suggests that there are \\'N\\' repeated blocks for the encoder and decoder, indicating a scalable architecture that can handle different complexities by increasing N.\\n\\nOverall, the diagram visually represents the flow of data through an attention-based architecture, highlighting the key operations and structural elements that facilitate learning from sequential data.',\n",
       " 'The image is a flowchart that illustrates a sequence of operations typically used in a matrix multiplication context, possibly related to the architecture of neural networks or attention mechanisms.\\n\\n1. **Components**: The diagram includes several labeled boxes arranged vertically, interconnected by arrows:\\n   - At the top, there is a box labeled \"MatMul,\" suggesting it represents matrix multiplication.\\n   - Below it is a rectangular box labeled \"SoftMax,\" which typically refers to a function that converts raw scores into probabilities.\\n   - The third box is \"Mask (opt.),\" possibly indicating an optional masking operation that is often used in contexts like attention mechanisms to ignore certain parts of the input.\\n   - The next labeled box is \"Scale,\" which implies a scaling operation, often used to adjust the output of matrix multiplications.\\n   - Lastly, there is another \"MatMul\" box at the bottom, which is involved in a second matrix multiplication operation.\\n\\n2. **Inputs and Outputs**: \\n   - Arrows mark the flow of data through the process, with \"Q\" (Query), \"K\" (Key), and \"V\" (Value) represented at the bottom, indicating they are likely inputs to the matrix multiplication operations.\\n   - The arrows leading into and out of the boxes illustrate the progression from inputs through various transformations (SoftMax, Mask, Scale) to outputs.\\n\\n3. **Color Scheme**: The boxes have distinct pastel colors—purple, green, pink, yellow—adding clarity to different operations and helping in visually distinguishing each component of the methodology.\\n\\nOverall, the image succinctly summarizes a complex series of steps in performing operations on data, which is typical in methodologies for factorial complete plans of experiments, especially in computational contexts like machine learning and data analysis.',\n",
       " 'The image is a flowchart or diagram representing the architecture of a scaled dot-product attention mechanism, commonly used in machine learning and neural networks, especially in models like Transformers.\\n\\n### Detailed Description:\\n\\n1. **Central Element:**\\n   - In the center of the diagram, there is a large block labeled \"Scaled Dot-Product Attention.\" This is the main component of the mechanism being illustrated.\\n\\n2. **Input Components:**\\n   - Below the central block, there are three separate blocks labeled \"V,\" \"K,\" and \"Q.\" These stand for \"Value,\" \"Key,\" and \"Query,\" respectively. Each of these blocks is connected to the attention mechanism, indicating that they feed into it.\\n   - Each of the input blocks has a smaller \"Linear\" block above it, suggesting that some linear transformation is applied to the incoming data before they enter the attention mechanism.\\n\\n3. **Concatenation:**\\n   - Above the \"Scaled Dot-Product Attention\" block, there\\'s an additional block labeled \"Concat,\" indicating that there\\'s a concatenation step happening after the linear transformations. This typically means that the outputs from the linear transformations of \"V,\" \"K,\" and \"Q\" are combined into a single representation that feeds into the attention mechanism.\\n\\n4. **Arrows and Flow:**\\n   - There are directional arrows connecting the blocks. The arrows show the flow of data from \"V,\" \"K,\" and \"Q\" through their respective linear transformations, into the concatenation block, and finally into the scaled dot-product attention block. This indicates how the information moves through the different components of the architecture.\\n\\n5. **Additional Features:**\\n   - The use of different colors may highlight the distinct roles of the blocks, with the central attention block in purple and the linear transformations in a lighter shade, helping to differentiate the components visually.\\n\\nOverall, the diagram succinctly outlines the structure and flow of information within a scaled dot-product attention mechanism, illustrating how input features are transformed and combined to facilitate attention-based processing in machine learning models.']"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_summaries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "import uuid\n",
    "from qdrant_client import QdrantClient\n",
    "from langchain_qdrant import QdrantVectorStore\n",
    "from langchain.storage import InMemoryStore\n",
    "from langchain.schema.document import Document\n",
    "from langchain_ollama import OllamaEmbeddings  # Vous pouvez remplacer par OpenAIEmbeddings si besoin\n",
    "from langchain.retrievers.multi_vector import MultiVectorRetriever\n",
    "from langchain.embeddings import OpenAIEmbeddings\n",
    "\n",
    "# Initialiser le client Qdrant (ici, on suppose un serveur local sur le port 6333)\n",
    "qdrant_client = QdrantClient(host=\"localhost\", port=6333)\n",
    "\n",
    "# Définir le nom de la collection\n",
    "collection_name = \"multi_modal_rag_final\"\n",
    "vector_size =1536\n",
    "# Créer (ou recréer) la collection dans Qdrant\n",
    "\n",
    "qdrant_client.create_collection(\n",
    "    collection_name=collection_name,\n",
    "    vectors_config={\"size\": vector_size, \"distance\": \"Cosine\"},\n",
    ")\n",
    "\n",
    "# Choix de la fonction d'embedding :\n",
    "# Pour utiliser OllamaEmbeddings :\n",
    "embedding_function = OpenAIEmbeddings()\n",
    "# Pour utiliser OpenAIEmbeddings, décommentez la ligne suivante et commentez la précédente :\n",
    "# embedding_function = OpenAIEmbeddings()\n",
    "\n",
    "# Créer la vectorstore avec QdrantVectorStore\n",
    "vectorstore = QdrantVectorStore(\n",
    "    client=qdrant_client,\n",
    "    collection_name=collection_name,\n",
    "    embedding=embedding_function\n",
    ")\n",
    "\n",
    "# Couche de stockage en mémoire pour les documents parents\n",
    "store = InMemoryStore()\n",
    "id_key = \"doc_id\"\n",
    "\n",
    "# Créer le retriever multi-vecteur\n",
    "retriever = MultiVectorRetriever(\n",
    "    vectorstore=vectorstore,\n",
    "    docstore=store,\n",
    "    id_key=id_key,\n",
    ")\n",
    "\n",
    "# --- Ajout des textes ---\n",
    "# (les variables texts et text_summaries doivent être définies au préalable)\n",
    "doc_ids = [str(uuid.uuid4()) for _ in texts]\n",
    "summary_texts = [\n",
    "    Document(page_content=summary, metadata={id_key: doc_ids[i]})\n",
    "    for i, summary in enumerate(text_summaries)\n",
    "]\n",
    "retriever.vectorstore.add_documents(summary_texts)\n",
    "retriever.docstore.mset(list(zip(doc_ids, texts)))\n",
    "\n",
    "# --- Ajout des tableaux ---\n",
    "# (les variables tables et table_summaries doivent être définies au préalable)\n",
    "table_ids = [str(uuid.uuid4()) for _ in tables]\n",
    "summary_tables = [\n",
    "    Document(page_content=summary, metadata={id_key: table_ids[i]})\n",
    "    for i, summary in enumerate(table_summaries)\n",
    "]\n",
    "retriever.vectorstore.add_documents(summary_tables)\n",
    "retriever.docstore.mset(list(zip(table_ids, tables)))\n",
    "\n",
    "# --- Ajout des résumés d'images ---\n",
    "# (les variables images et image_summaries doivent être définies au préalable)\n",
    "img_ids = [str(uuid.uuid4()) for _ in images]\n",
    "summary_img = [\n",
    "    Document(page_content=summary, metadata={id_key: img_ids[i]})\n",
    "    for i, summary in enumerate(image_summaries)\n",
    "]\n",
    "retriever.vectorstore.add_documents(summary_img)\n",
    "retriever.docstore.mset(list(zip(img_ids, images)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['1055f2ba-1d94-4d4e-8f7e-1b312f5153bc',\n",
       " '6a677a93-3256-4458-a344-f570d488d287',\n",
       " 'dd8b8ca1-0cb1-4b65-a257-ec7733324ced']"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "img_ids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = retriever.invoke(\n",
    "    \"who are the authors of the paper?\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "big\n",
      "\n",
      "6\n",
      "\n",
      "1024\n",
      "\n",
      "4096\n",
      "\n",
      "16\n",
      "\n",
      "0.3\n",
      "\n",
      "300K 4.33\n",
      "\n",
      "26.4\n",
      "\n",
      "In Table 3 rows (B), we observe that reducing the attention key size dk hurts model quality. This suggests that determining compatibility is not easy and that a more sophisticated compatibility function than dot product may be beneﬁcial. We further observe in rows (C) and (D) that, as expected, bigger models are better, and dropout is very helpful in avoiding over-ﬁtting. In row (E) we replace our sinusoidal positional encoding with learned positional embeddings [8], and observe nearly identical results to the base model.\n",
      "\n",
      "7 Conclusion\n",
      "\n",
      "In this work, we presented the Transformer, the ﬁrst sequence transduction model based entirely on attention, replacing the recurrent layers most commonly used in encoder-decoder architectures with multi-headed self-attention.\n",
      "\n",
      "For translation tasks, the Transformer can be trained signiﬁcantly faster than architectures based on recurrent or convolutional layers. On both WMT 2014 English-to-German and WMT 2014 English-to-French translation tasks, we achieve a new state of the art. In the former task our best model outperforms even all previously reported ensembles.\n",
      "\n",
      "We are excited about the future of attention-based models and plan to apply them to other tasks. We plan to extend the Transformer to problems involving input and output modalities other than text and to investigate local, restricted attention mechanisms to efﬁciently handle large inputs and outputs such as images, audio and video. Making generation less sequential is another research goals of ours.\n",
      "\n",
      "The code we used to train and evaluate our models is available at https://github.com/ tensorflow/tensor2tensor.\n",
      "\n",
      "Acknowledgements We are grateful to Nal Kalchbrenner and Stephan Gouws for their fruitful comments, corrections and inspiration.\n",
      "\n",
      "9\n",
      "\n",
      "213\n",
      "\n",
      "References\n",
      "\n",
      "[1] Jimmy Lei Ba, Jamie Ryan Kiros, and Geoffrey E Hinton. Layer normalization. arXiv preprint arXiv:1607.06450, 2016.\n",
      "\n",
      "[2] Dzmitry Bahdanau, Kyunghyun Cho, and Yoshua Bengio. Neural machine translation by jointly learning to align and translate. CoRR, abs/1409.0473, 2014.\n",
      "\n",
      "[3] Denny Britz, Anna Goldie, Minh-Thang Luong, and Quoc V. Le. Massive exploration of neural machine translation architectures. CoRR, abs/1703.03906, 2017.\n",
      "\n",
      "[4] Jianpeng Cheng, Li Dong, and Mirella Lapata. Long short-term memory-networks for machine reading. arXiv preprint arXiv:1601.06733, 2016.\n",
      "\n",
      "[5] Kyunghyun Cho, Bart van Merrienboer, Caglar Gulcehre, Fethi Bougares, Holger Schwenk, and Yoshua Bengio. Learning phrase representations using rnn encoder-decoder for statistical machine translation. CoRR, abs/1406.1078, 2014.\n",
      "\n",
      "[6] Francois Chollet. Xception: Deep learning with depthwise separable convolutions. arXiv preprint arXiv:1610.02357, 2016.\n",
      "\n",
      "[7] Junyoung Chung, Çaglar Gülçehre, Kyunghyun Cho, and Yoshua Bengio. Empirical evaluation of gated recurrent neural networks on sequence modeling. CoRR, abs/1412.3555, 2014.\n",
      "\n",
      "[8] Jonas Gehring, Michael Auli, David Grangier, Denis Yarats, and Yann N. Dauphin. Convolu- tional sequence to sequence learning. arXiv preprint arXiv:1705.03122v2, 2017.\n",
      "\n",
      "[9] Alex Graves. Generating sequences with recurrent neural networks. arXiv preprint arXiv:1308.0850, 2013.\n",
      "\n",
      "[10] Kaiming He, Xiangyu Zhang, Shaoqing Ren, and Jian Sun. Deep residual learning for im- age recognition. In Proceedings of the IEEE Conference on Computer Vision and Pattern Recognition, pages 770–778, 2016.\n",
      "\n",
      "[11] Sepp Hochreiter, Yoshua Bengio, Paolo Frasconi, and Jürgen Schmidhuber. Gradient ﬂow in recurrent nets: the difﬁculty of learning long-term dependencies, 2001.\n",
      "\n",
      "[12] Sepp Hochreiter and Jürgen Schmidhuber. Long short-term memory. Neural computation, 9(8):1735–1780, 1997.\n",
      "\n",
      "[13] Rafal Jozefowicz, Oriol Vinyals, Mike Schuster, Noam Shazeer, and Yonghui Wu. Exploring the limits of language modeling. arXiv preprint arXiv:1602.02410, 2016.\n",
      "\n",
      "[14] Łukasz Kaiser and Ilya Sutskever. Neural GPUs learn algorithms. In International Conference on Learning Representations (ICLR), 2016.\n",
      "\n",
      "[15] Nal Kalchbrenner, Lasse Espeholt, Karen Simonyan, Aaron van den Oord, Alex Graves, and Ko- ray Kavukcuoglu. Neural machine translation in linear time. arXiv preprint arXiv:1610.10099v2, 2017.\n",
      "\n",
      "[16] Yoon Kim, Carl Denton, Luong Hoang, and Alexander M. Rush. Structured attention networks. In International Conference on Learning Representations, 2017.\n",
      "\n",
      "[17] Diederik Kingma and Jimmy Ba. Adam: A method for stochastic optimization. In ICLR, 2015.\n",
      "\n",
      "[18] Oleksii Kuchaiev and Boris Ginsburg. Factorization tricks for LSTM networks. arXiv preprint arXiv:1703.10722, 2017.\n",
      "\n",
      "[19] Zhouhan Lin, Minwei Feng, Cicero Nogueira dos Santos, Mo Yu, Bing Xiang, Bowen Zhou, and Yoshua Bengio. A structured self-attentive sentence embedding. arXiv preprint arXiv:1703.03130, 2017.\n",
      "\n",
      "[20] Samy Bengio Łukasz Kaiser. Can active memory replace attention? In Advances in Neural Information Processing Systems, (NIPS), 2016.\n",
      "\n",
      "10\n",
      "\n",
      "[21] Minh-Thang Luong, Hieu Pham, and Christopher D Manning. Effective approaches to attention- based neural machine translation. arXiv preprint arXiv:1508.04025, 2015.\n",
      "\n",
      "[22] Ankur Parikh, Oscar Täckström, Dipanjan Das, and Jakob Uszkoreit. A decomposable attention model. In Empirical Methods in Natural Language Processing, 2016.\n",
      "\n",
      "[23] Romain Paulus, Caiming Xiong, and Richard Socher. A deep reinforced model for abstractive summarization. arXiv preprint arXiv:1705.04304, 2017.\n",
      "\n",
      "[24] Oﬁr Press and Lior Wolf. Using the output embedding to improve language models. arXiv preprint arXiv:1608.05859, 2016.\n",
      "\n",
      "[25] Rico Sennrich, Barry Haddow, and Alexandra Birch. Neural machine translation of rare words with subword units. arXiv preprint arXiv:1508.07909, 2015.\n",
      "\n",
      "[26] Noam Shazeer, Azalia Mirhoseini, Krzysztof Maziarz, Andy Davis, Quoc Le, Geoffrey Hinton, and Jeff Dean. Outrageously large neural networks: The sparsely-gated mixture-of-experts layer. arXiv preprint arXiv:1701.06538, 2017.\n",
      "\n",
      "[27] Nitish Srivastava, Geoffrey E Hinton, Alex Krizhevsky, Ilya Sutskever, and Ruslan Salakhutdi- nov. Dropout: a simple way to prevent neural networks from overﬁtting. Journal of Machine Learning Research, 15(1):1929–1958, 2014.\n",
      "\n",
      "[28] Sainbayar Sukhbaatar, arthur szlam, Jason Weston, and Rob Fergus. End-to-end memory networks. In C. Cortes, N. D. Lawrence, D. D. Lee, M. Sugiyama, and R. Garnett, editors, Advances in Neural Information Processing Systems 28, pages 2440–2448. Curran Associates, Inc., 2015.\n",
      "\n",
      "[29] Ilya Sutskever, Oriol Vinyals, and Quoc VV Le. Sequence to sequence learning with neural networks. In Advances in Neural Information Processing Systems, pages 3104–3112, 2014.\n",
      "\n",
      "[30] Christian Szegedy, Vincent Vanhoucke, Sergey Ioffe, Jonathon Shlens, and Zbigniew Wojna. Rethinking the inception architecture for computer vision. CoRR, abs/1512.00567, 2015.\n",
      "\n",
      "[31] Yonghui Wu, Mike Schuster, Zhifeng Chen, Quoc V Le, Mohammad Norouzi, Wolfgang Macherey, Maxim Krikun, Yuan Cao, Qin Gao, Klaus Macherey, et al. Google’s neural machine translation system: Bridging the gap between human and machine translation. arXiv preprint arXiv:1609.08144, 2016.\n",
      "\n",
      "[32] Jie Zhou, Ying Cao, Xuguang Wang, Peng Li, and Wei Xu. Deep recurrent models with fast-forward connections for neural machine translation. CoRR, abs/1606.04199, 2016.\n",
      "\n",
      "11\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "2 Background\n",
      "\n",
      "The goal of reducing sequential computation also forms the foundation of the Extended Neural GPU [20], ByteNet [15] and ConvS2S [8], all of which use convolutional neural networks as basic building block, computing hidden representations in parallel for all input and output positions. In these models, the number of operations required to relate signals from two arbitrary input or output positions grows in the distance between positions, linearly for ConvS2S and logarithmically for ByteNet. This makes it more difﬁcult to learn dependencies between distant positions [11]. In the Transformer this is reduced to a constant number of operations, albeit at the cost of reduced effective resolution due to averaging attention-weighted positions, an effect we counteract with Multi-Head Attention as described in section 3.2.\n",
      "\n",
      "Self-attention, sometimes called intra-attention is an attention mechanism relating different positions of a single sequence in order to compute a representation of the sequence. Self-attention has been used successfully in a variety of tasks including reading comprehension, abstractive summarization, textual entailment and learning task-independent sentence representations [4, 22, 23, 19].\n",
      "\n",
      "End-to-end memory networks are based on a recurrent attention mechanism instead of sequence- aligned recurrence and have been shown to perform well on simple-language question answering and language modeling tasks [28].\n",
      "\n",
      "To the best of our knowledge, however, the Transformer is the ﬁrst transduction model relying entirely on self-attention to compute representations of its input and output without using sequence- aligned RNNs or convolution. In the following sections, we will describe the Transformer, motivate self-attention and discuss its advantages over models such as [14, 15] and [8].\n",
      "\n",
      "3 Model Architecture\n",
      "\n",
      "Most competitive neural sequence transduction models have an encoder-decoder structure [5, 2, 29]. Here, the encoder maps an input sequence of symbol representations (x1,...,xn) to a sequence of continuous representations z = (z1,...,zn). Given z, the decoder then generates an output sequence (y1,...,ym) of symbols one element at a time. At each step the model is auto-regressive [9], consuming the previously generated symbols as additional input when generating the next.\n",
      "\n",
      "The Transformer follows this overall architecture using stacked self-attention and point-wise, fully connected layers for both the encoder and decoder, shown in the left and right halves of Figure 1, respectively.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "Attention Is All You Need\n",
      "\n",
      "Ashish Vaswani∗ Google Brain avaswani@google.com\n",
      "\n",
      "Noam Shazeer∗ Google Brain noam@google.com\n",
      "\n",
      "Niki Parmar∗\n",
      "\n",
      "Google Research nikip@google.com\n",
      "\n",
      "Jakob Uszkoreit∗ Google Research usz@google.com\n",
      "\n",
      "Llion Jones∗ Google Research llion@google.com\n",
      "\n",
      "Aidan N. Gomez∗ † University of Toronto aidan@cs.toronto.edu\n",
      "\n",
      "Łukasz Kaiser∗ Google Brain lukaszkaiser@google.com\n",
      "\n",
      "Illia Polosukhin∗ ‡\n",
      "\n",
      "illia.polosukhin@gmail.com\n",
      "\n",
      "Abstract\n",
      "\n",
      "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring signiﬁcantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English- to-German translation task, improving over the existing best results, including ensembles, by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.0 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature.\n",
      "\n",
      "1 Introduction\n",
      "\n",
      "Recurrent neural networks, long short-term memory [12] and gated recurrent [7] neural networks in particular, have been ﬁrmly established as state of the art approaches in sequence modeling and transduction problems such as language modeling and machine translation [29, 2, 5]. Numerous efforts have since continued to push the boundaries of recurrent language models and encoder-decoder architectures [31, 21, 13].\n",
      "\n",
      "∗Equal contribution. Listing order is random. Jakob proposed replacing RNNs with self-attention and started the effort to evaluate this idea. Ashish, with Illia, designed and implemented the ﬁrst Transformer models and has been crucially involved in every aspect of this work. Noam proposed scaled dot-product attention, multi-head attention and the parameter-free position representation and became the other person involved in nearly every detail. Niki designed, implemented, tuned and evaluated countless model variants in our original codebase and tensor2tensor. Llion also experimented with novel model variants, was responsible for our initial codebase, and efﬁcient inference and visualizations. Lukasz and Aidan spent countless long days designing various parts of and implementing tensor2tensor, replacing our earlier codebase, greatly improving results and massively accelerating our research.\n",
      "\n",
      "†Work performed while at Google Brain.\n",
      "\n",
      "‡Work performed while at Google Research.\n",
      "\n",
      "31st Conference on Neural Information Processing Systems (NIPS 2017), Long Beach, CA, USA.\n",
      "\n",
      "Recurrent models typically factor computation along the symbol positions of the input and output sequences. Aligning the positions to steps in computation time, they generate a sequence of hidden states ht, as a function of the previous hidden state ht−1 and the input for position t. This inherently sequential nature precludes parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit batching across examples. Recent work has achieved signiﬁcant improvements in computational efﬁciency through factorization tricks [18] and conditional computation [26], while also improving model performance in case of the latter. The fundamental constraint of sequential computation, however, remains.\n",
      "\n",
      "Attention mechanisms have become an integral part of compelling sequence modeling and transduc- tion models in various tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences [2, 16]. In all but a few cases [22], however, such attention mechanisms are used in conjunction with a recurrent network.\n",
      "\n",
      "In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. The Transformer allows for signiﬁcantly more parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve hours on eight P100 GPUs.\n",
      "\n",
      "--------------------------------------------------------------------------------\n",
      "3.2.1 Scaled Dot-Product Attention\n",
      "\n",
      "We call our particular attention \"Scaled Dot-Product Attention\" (Figure 2). The input consists of queries and keys of dimension dk, and values of dimension dv. We compute the dot products of the\n",
      "\n",
      "3\n",
      "\n",
      "Scaled Dot-Product Attention\n",
      "\n",
      "Multi-Head Attention\n",
      "\n",
      "Linear\n",
      "\n",
      "Figure 2: (left) Scaled Dot-Product Attention. (right) Multi-Head Attention consists of several attention layers running in parallel.\n",
      "\n",
      "√\n",
      "\n",
      "query with all keys, divide each by dk, and apply a softmax function to obtain the weights on the values.\n",
      "\n",
      "In practice, we compute the attention function on a set of queries simultaneously, packed together into a matrix Q. The keys and values are also packed together into matrices K and V . We compute the matrix of outputs as:\n",
      "\n",
      "Attention(Q,K,V ) = softmax( QKT √ dk )V (1)\n",
      "\n",
      "The two most commonly used attention functions are additive attention [2], and dot-product (multi- plicative) attention. Dot-product attention is identical to our algorithm, except for the scaling factor 1√ of . Additive attention computes the compatibility function using a feed-forward network with dk a single hidden layer. While the two are similar in theoretical complexity, dot-product attention is much faster and more space-efﬁcient in practice, since it can be implemented using highly optimized matrix multiplication code.\n",
      "\n",
      "While for small values of dk the two mechanisms perform similarly, additive attention outperforms dot product attention without scaling for larger values of dk [3]. We suspect that for large values of dk, the dot products grow large in magnitude, pushing the softmax function into regions where it has extremely small gradients 4. To counteract this effect, we scale the dot products by 1√ . dk\n",
      "\n",
      "3.2.2 Multi-Head Attention\n",
      "\n",
      "Instead of performing a single attention function with dmodel-dimensional keys, values and queries, we found it beneﬁcial to linearly project the queries, keys and values h times with different, learned linear projections to dk, dk and dv dimensions, respectively. On each of these projected versions of queries, keys and values we then perform the attention function in parallel, yielding dv-dimensional output values. These are concatenated and once again projected, resulting in the ﬁnal values, as depicted in Figure 2.\n",
      "\n",
      "Multi-head attention allows the model to jointly attend to information from different representation subspaces at different positions. With a single attention head, averaging inhibits this.\n",
      "\n",
      "‘To illustrate why the dot products get large, assume that the components of q and k are independent random variables with mean 0 and variance 1. Then their dot product, g -k = ves, qiki, has mean 0 and variance dx.\n",
      "\n",
      "4\n",
      "\n",
      "MultiHead(Q,K,V ) = Concat(head1,...,headh)W O where headi = Attention(QW Q i ,KW K i ,V W V i )\n",
      "\n",
      "Where the projections are parameter matrices W Q and W O ∈ Rhdv×dmodel. i ∈ Rdmodel×dk, W K i ∈ Rdmodel×dk, W V i ∈ Rdmodel×dv\n",
      "\n",
      "In this work we employ h = 8 parallel attention layers, or heads. For each of these we use dk = dv = dmodel/h = 64. Due to the reduced dimension of each head, the total computational cost is similar to that of single-head attention with full dimensionality.\n",
      "\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "for doc in docs:\n",
    "    print(str(doc) + \"\\n\\n\" + \"-\" * 80)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_core.runnables import RunnablePassthrough, RunnableLambda\n",
    "from langchain_core.messages import SystemMessage, HumanMessage\n",
    "from langchain_openai import ChatOpenAI\n",
    "from langchain_ollama import ChatOllama\n",
    "from base64 import b64decode\n",
    "\n",
    "\n",
    "def parse_docs(docs):\n",
    "    \"\"\"Split base64-encoded images and texts\"\"\"\n",
    "    b64 = []\n",
    "    text = []\n",
    "    for doc in docs:\n",
    "        try:\n",
    "            b64decode(doc)\n",
    "            b64.append(doc)\n",
    "        except Exception as e:\n",
    "            text.append(doc)\n",
    "    return {\"images\": b64, \"texts\": text}\n",
    "\n",
    "\n",
    "def build_prompt(kwargs):\n",
    "\n",
    "    docs_by_type = kwargs[\"context\"]\n",
    "    user_question = kwargs[\"question\"]\n",
    "\n",
    "    context_text = \"\"\n",
    "    if len(docs_by_type[\"texts\"]) > 0:\n",
    "        for text_element in docs_by_type[\"texts\"]:\n",
    "            context_text += text_element.text\n",
    "\n",
    "    # construct prompt with context (including images)\n",
    "    prompt_template = f\"\"\"\n",
    "    Answer the question based only on the following context, which can include text, tables, and the below image.\n",
    "    Context: {context_text}\n",
    "    Question: {user_question}\n",
    "    \"\"\"\n",
    "\n",
    "    prompt_content = [{\"type\": \"text\", \"text\": prompt_template}]\n",
    "\n",
    "    if len(docs_by_type[\"images\"]) > 0:\n",
    "        for image in docs_by_type[\"images\"]:\n",
    "            prompt_content.append(\n",
    "                {\n",
    "                    \"type\": \"image_url\",\n",
    "                    \"image_url\": {\"url\": f\"data:image/jpeg;base64,{image}\"},\n",
    "                }\n",
    "            )\n",
    "\n",
    "    return ChatPromptTemplate.from_messages(\n",
    "        [\n",
    "            HumanMessage(content=prompt_content),\n",
    "        ]\n",
    "    )\n",
    "\n",
    "\n",
    "chain = (\n",
    "    {\n",
    "        \"context\": retriever | RunnableLambda(parse_docs),\n",
    "        \"question\": RunnablePassthrough(),\n",
    "    }\n",
    "    | RunnableLambda(build_prompt)\n",
    "    | ChatOllama(base_url=\"172.20.48.1:11434\", model=\"llama3.2\")\n",
    "    | StrOutputParser()\n",
    ")\n",
    "# ChatOpenAI(model=\"gpt-4o-mini\")\n",
    "chain_with_sources = {\n",
    "    \"context\": retriever | RunnableLambda(parse_docs),\n",
    "    \"question\": RunnablePassthrough(),\n",
    "} | RunnablePassthrough().assign(\n",
    "    response=(\n",
    "        RunnableLambda(build_prompt)\n",
    "        | ChatOllama(base_url=\"172.20.48.1:11434\", model=\"llama3.2\")\n",
    "        | StrOutputParser()\n",
    "    )\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Voici un résumé du papier :\n",
      "\n",
      "**Introduction**\n",
      "\n",
      "Le papier présente un nouveau modèle de traitement automatique des langues appelé Transformer, qui utilise l'attention pour traiter les séquences de mots.\n",
      "\n",
      "**Architecture du modèle**\n",
      "\n",
      "Le modèle est composé de deux sous-parties : l'encodage et le décodage. L'encodage est constitué d'un stack de 6 couches identiques, chaque couche étant composée de deux sous-couches : une sous-couche de multi-head attention et une sous-couche de réseau à réseaux connexés (FFNN). Le décodage est également constitué d'un stack de 6 couches identiques, mais avec une troisième sous-couche de multi-head attention sur les outputs du stack encodant.\n",
      "\n",
      "**Attention**\n",
      "\n",
      "L'attention est un fonctionnement qui permet à chaque mot de prendre en compte tous les mots de la phrase pour faire le choix de la préposition la plus appropriée. L'attention est calculée en utilisant une combinaison linéaire des valeurs et des clés, où les clés sont les vectors représentant les mots.\n",
      "\n",
      "**Modèle variée**\n",
      "\n",
      "Le modèle présenté dans le papier peut être modifié pour évaluer l'importance de différentes parties du modèle. Les expériences présentées montrent que la modification de certaines parties du modèle peut avoir un impact significatif sur la performance du modèle.\n",
      "\n",
      "**Résultats**\n",
      "\n",
      "Les résultats présentés dans le papier sont positifs, avec une meilleure performance que les modèles traditionnels en traitement automatique des langues. Le modèle présenté est capable de gérer des longueurs de phrase et des contextes plus complexes que les modèles traditionnels.\n",
      "\n",
      "En résumé, ce papier présente un nouveau modèle de traitement automatique des langues qui utilise l'attention pour traiter les séquences de mots. Les résultats montrent que le modèle présenté est capable de gérer des longueurs de phrase et des contextes plus complexes que les modèles traditionnels.\n"
     ]
    }
   ],
   "source": [
    "response = chain.invoke(\n",
    "    \"résume moi le papier ?\"\n",
    ")\n",
    "\n",
    "print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Response: Unfortunately, I'm a text-based AI and cannot display images directly. However, I can provide you with a textual representation of the Transformer architecture based on Figure 1 provided in the context.\n",
      "\n",
      "The Transformer architecture consists of an encoder and a decoder stack, each composed of identical layers. Here is a simplified diagram of the model:\n",
      "\n",
      "**Encoder:**\n",
      "\n",
      "* Input Embedding (Embedding Inputs)\n",
      "* Positional Encoding\n",
      "* Layer 1:\n",
      " + Multi-Head Self-Attention Mechanism\n",
      " + Residual Connection and Layer Normalization (LayerNorm)\n",
      "* Layer 2:\n",
      " + Simple, Position-Wise Fully Connected Feed-Forward Network\n",
      " + Residual Connection and Layer Normalization (LayerNorm)\n",
      "\n",
      "**Decoder:**\n",
      "\n",
      "* Input Embedding (Embedding Outputs)\n",
      "* Positional Encoding\n",
      "* Layer 1:\n",
      " + Multi-Head Self-Attention Mechanism\n",
      " + Residual Connection and Layer Normalization (LayerNorm)\n",
      "* Layer 2:\n",
      " + Simple, Position-Wise Fully Connected Feed-Forward Network\n",
      " + Residual Connection and Layer Normalization (LayerNorm)\n",
      "* Additional Sub-Layer:\n",
      " + Multi-Head Attention over the Output of the Encoder Stack\n",
      " + Residual Connection and Layer Normalization (LayerNorm)\n",
      "\n",
      "Here is a more detailed representation of the sub-layers:\n",
      "\n",
      "* Multi-Head Self-Attention Mechanism: \n",
      "  - Query (Q) and Key (K)\n",
      "  - Compute Attention Weights and Apply to Value (V)\n",
      "  - Output as Weighted Sum\n",
      "* Simple, Position-Wise Fully Connected Feed-Forward Network:\n",
      "  - Input Embedding\n",
      "  - ReLU Activation Function\n",
      "  - Residual Connection and Layer Normalization (LayerNorm)\n",
      "* Multi-Head Attention over the Output of the Encoder Stack:\n",
      "  - Query (Q) from Decoder Outputs\n",
      "  - Key (K) and Value (V) from Encoder Outputs\n",
      "  - Compute Attention Weights and Apply to Value (V)\n",
      "  - Output as Weighted Sum\n",
      "\n",
      "Note that this is a simplified representation, and you can refer to the original paper or supplementary materials for more details on the architecture.\n",
      "\n",
      "\n",
      "Context:\n",
      "Attention Is All You Need\n",
      "\n",
      "Ashish Vaswani∗ Google Brain avaswani@google.com\n",
      "\n",
      "Noam Shazeer∗ Google Brain noam@google.com\n",
      "\n",
      "Niki Parmar∗\n",
      "\n",
      "Google Research nikip@google.com\n",
      "\n",
      "Jakob Uszkoreit∗ Google Research usz@google.com\n",
      "\n",
      "Llion Jones∗ Google Research llion@google.com\n",
      "\n",
      "Aidan N. Gomez∗ † University of Toronto aidan@cs.toronto.edu\n",
      "\n",
      "Łukasz Kaiser∗ Google Brain lukaszkaiser@google.com\n",
      "\n",
      "Illia Polosukhin∗ ‡\n",
      "\n",
      "illia.polosukhin@gmail.com\n",
      "\n",
      "Abstract\n",
      "\n",
      "The dominant sequence transduction models are based on complex recurrent or convolutional neural networks that include an encoder and a decoder. The best performing models also connect the encoder and decoder through an attention mechanism. We propose a new simple network architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two machine translation tasks show these models to be superior in quality while being more parallelizable and requiring signiﬁcantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014 English- to-German translation task, improving over the existing best results, including ensembles, by over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new single-model state-of-the-art BLEU score of 41.0 after training for 3.5 days on eight GPUs, a small fraction of the training costs of the best models from the literature.\n",
      "\n",
      "1 Introduction\n",
      "\n",
      "Recurrent neural networks, long short-term memory [12] and gated recurrent [7] neural networks in particular, have been ﬁrmly established as state of the art approaches in sequence modeling and transduction problems such as language modeling and machine translation [29, 2, 5]. Numerous efforts have since continued to push the boundaries of recurrent language models and encoder-decoder architectures [31, 21, 13].\n",
      "\n",
      "∗Equal contribution. Listing order is random. Jakob proposed replacing RNNs with self-attention and started the effort to evaluate this idea. Ashish, with Illia, designed and implemented the ﬁrst Transformer models and has been crucially involved in every aspect of this work. Noam proposed scaled dot-product attention, multi-head attention and the parameter-free position representation and became the other person involved in nearly every detail. Niki designed, implemented, tuned and evaluated countless model variants in our original codebase and tensor2tensor. Llion also experimented with novel model variants, was responsible for our initial codebase, and efﬁcient inference and visualizations. Lukasz and Aidan spent countless long days designing various parts of and implementing tensor2tensor, replacing our earlier codebase, greatly improving results and massively accelerating our research.\n",
      "\n",
      "†Work performed while at Google Brain.\n",
      "\n",
      "‡Work performed while at Google Research.\n",
      "\n",
      "31st Conference on Neural Information Processing Systems (NIPS 2017), Long Beach, CA, USA.\n",
      "\n",
      "Recurrent models typically factor computation along the symbol positions of the input and output sequences. Aligning the positions to steps in computation time, they generate a sequence of hidden states ht, as a function of the previous hidden state ht−1 and the input for position t. This inherently sequential nature precludes parallelization within training examples, which becomes critical at longer sequence lengths, as memory constraints limit batching across examples. Recent work has achieved signiﬁcant improvements in computational efﬁciency through factorization tricks [18] and conditional computation [26], while also improving model performance in case of the latter. The fundamental constraint of sequential computation, however, remains.\n",
      "\n",
      "Attention mechanisms have become an integral part of compelling sequence modeling and transduc- tion models in various tasks, allowing modeling of dependencies without regard to their distance in the input or output sequences [2, 16]. In all but a few cases [22], however, such attention mechanisms are used in conjunction with a recurrent network.\n",
      "\n",
      "In this work we propose the Transformer, a model architecture eschewing recurrence and instead relying entirely on an attention mechanism to draw global dependencies between input and output. The Transformer allows for signiﬁcantly more parallelization and can reach a new state of the art in translation quality after being trained for as little as twelve hours on eight P100 GPUs.\n",
      "Page number:  1\n",
      "\n",
      "--------------------------------------------------\n",
      "\n",
      "3.1 Encoder and Decoder Stacks\n",
      "\n",
      "Encoder: The encoder is composed of a stack of N = 6 identical layers. Each layer has two sub-layers. The ﬁrst is a multi-head self-attention mechanism, and the second is a simple, position-\n",
      "\n",
      "2\n",
      "\n",
      "Output Probabilities Add & Norm Feed Forward Add & Norm Multi-Head Attention a, Add & Norm Add & Norm Feed Forward Nx | Cag Norm) Add & Norm Masked Multi-Head Multi-Head Attention Attention ae a, Lt Positional Positional Encoding CY © @ Encoding Input Output Embedding Embedding Inputs Outputs (shifted right)\n",
      "\n",
      "Figure 1: The Transformer - model architecture.\n",
      "\n",
      "wise fully connected feed-forward network. We employ a residual connection [10] around each of the two sub-layers, followed by layer normalization [1]. That is, the output of each sub-layer is LayerNorm(x + Sublayer(x)), where Sublayer(x) is the function implemented by the sub-layer itself. To facilitate these residual connections, all sub-layers in the model, as well as the embedding layers, produce outputs of dimension dmodel = 512.\n",
      "\n",
      "Decoder: The decoder is also composed of a stack of N = 6 identical layers. In addition to the two sub-layers in each encoder layer, the decoder inserts a third sub-layer, which performs multi-head attention over the output of the encoder stack. Similar to the encoder, we employ residual connections around each of the sub-layers, followed by layer normalization. We also modify the self-attention sub-layer in the decoder stack to prevent positions from attending to subsequent positions. This masking, combined with fact that the output embeddings are offset by one position, ensures that the predictions for position i can depend only on the known outputs at positions less than i.\n",
      "\n",
      "3.2 Attention\n",
      "\n",
      "An attention function can be described as mapping a query and a set of key-value pairs to an output, where the query, keys, values, and output are all vectors. The output is computed as a weighted sum of the values, where the weight assigned to each value is computed by a compatibility function of the query with the corresponding key.\n",
      "Page number:  2\n",
      "\n",
      "--------------------------------------------------\n",
      "\n"
     ]
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 95284 characters]",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/jpeg": "[encoded data, 27564 characters]",
      "text/plain": [
       "<IPython.core.display.Image object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "response = chain_with_sources.invoke(\n",
    "    \"give me diagram of a transformer architecture\"\n",
    ")\n",
    "\n",
    "print(\"Response:\", response['response'])\n",
    "\n",
    "print(\"\\n\\nContext:\")\n",
    "for text in response['context']['texts']:\n",
    "    print(text.text)\n",
    "    print(\"Page number: \", text.metadata.page_number)\n",
    "    print(\"\\n\" + \"-\"*50 + \"\\n\")\n",
    "for image in response['context']['images']:\n",
    "    display_base64_image(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "for image in response['context']['images']:\n",
    "    display_base64_image(image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"The image represents a diagram of a transformer architecture commonly used in machine learning, particularly in natural language processing. It consists of two main sections—one for processing inputs and one for generating outputs. \\n\\n### Left Side (Input Processing):\\n- **Inputs** enter the model, which are then processed through:\\n  - **Input Embedding**: Converts input tokens into dense vector representations.\\n  - **Positional Encoding**: Adds information to the embeddings to retain the order of the inputs.\\n- The embedding goes through multiple layers (denoted as \\\\( N_x \\\\)) where each layer includes:\\n  - **Multi-Head Attention**: This mechanism allows the model to focus on different parts of the input sequence simultaneously, capturing contextual relationships.\\n  - **Add & Norm**: This operation adds the output of the attention layer to the original input followed by normalization, enhancing stability and training efficiency.\\n  - **Feed Forward**: A position-wise feed-forward layer that processes each position independently.\\n\\n### Right Side (Output Generation):\\n- The output side follows a similar structure, processing outputs shifted right:\\n  - **Output Embedding**: Converts the output tokens into embeddings similar to the input side.\\n  - **Masked Multi-Head Attention**: This prevents the model from attending to future tokens, which is crucial during decoding.\\n  - **Add & Norm**: Similar to the input side, it adds the attention output to the input followed by normalization.\\n  - The final layers include:\\n    - **Linear**: A linear transformation that converts the output into logits.\\n    - **Softmax**: Converts the logits into probabilities, allowing for interpretation as predicted class probabilities.\\n\\nAt the top, it indicates that the final output is a set of **Output Probabilities**, showcasing the model's predictions based on the processed input. The overall architecture emphasizes the flow of information through multiple layers, highlighting the complexity and power of the transformer model.\",\n",
       " 'The image presents a diagram showing a series of interconnected components, likely related to a process in machine learning or neural networks, specifically within the context of a transformer model. \\n\\nHere’s a detailed description:\\n\\n1. **Components**:\\n   - **MatMul**: At the top and bottom of the diagram, there are two boxes labeled \"MatMul.\" This likely refers to matrix multiplication operations, which are fundamental in many computational processes.\\n   \\n   - **SoftMax**: There is a box labeled \"SoftMax\" above the lower MatMul, indicating a function that converts logits into probabilities.\\n\\n   - **Mask (opt.)**: This element is positioned next to SoftMax, indicating a masking operation, which is often used to prevent certain values from affecting the output in calculations.\\n\\n   - **Scale**: Below the Mask, there\\'s a box labeled \"Scale,\" suggesting a step where values are scaled before further processing.\\n\\n2. **Arrows**: \\n   - There are directional arrows connecting these components, indicating the flow of data or operations from one box to the next.\\n   \\n   - Specifically, two arrows lead up to the top MatMul and down from the bottom MatMul, hinting at information being passed back and forth.\\n\\n3. **Inputs (Q, K, V)**: At the bottom, there are labels for \"Q,\" \"K,\" and \"V,\" likely representing queries, keys, and values, which are fundamental components in attention mechanisms within neural architectures.\\n\\nOverall, the diagram seems to illustrate a structured process for a functional workflow involving matrix operations and attention mechanisms, commonly seen in advanced machine learning architectures.',\n",
       " 'The image presents a diagram illustrating a component of a neural network, specifically focusing on the \"Scaled Dot-Product Attention\" mechanism. Here\\'s a detailed breakdown:\\n\\n1. **Structure**:\\n   - The diagram features a series of rectangular boxes, each labeled with specific functions.\\n   - The key components include three \"Linear\" boxes, representing different linear transformations applied to input data.\\n   - These are positioned vertically, leading into the central box labeled \"Scaled Dot-Product Attention,\" which is shaded in purple for emphasis.\\n\\n2. **Inputs**:\\n   - There are three inputs to the attention mechanism denoted by \"Q\" (Query), \"K\" (Key), and \"V\" (Value). \\n   - Each of these inputs flow into their respective \"Linear\" transformation boxes at the bottom of the diagram.\\n\\n3. **Concatenation**:\\n   - Above the \"Linear\" boxes, there is a box labeled \"Concat,\" indicating that the outputs from the linear transformations of Q, K, and V will be concatenated together after processing.\\n\\n4. **Flow of Information**:\\n   - There are arrows indicating the flow of information:\\n     - Arrows point from the \"Linear\" boxes for Q, K, and V towards the \"Concat\" box.\\n     - The output of the \"Concat\" box then flows into the \"Scaled Dot-Product Attention\" box.\\n     - Additional arrows suggest that output from the attention mechanism can further influence other processes or layers in the network.\\n\\n5. **Output**:\\n   - The entire process is aimed at producing a context-dependent output derived from the attention mechanism, which helps the model weigh the importance of different input elements based on their relationships.\\n\\nOverall, the diagram effectively outlines the functionality and flow of data within the scaled dot-product attention mechanism, crucial in various neural network models, especially in natural language processing tasks.']"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "image_summaries\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": ".venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
