{ "cells": [ { "cell_type": "markdown", "metadata": { "id": "Pdval7tUZwdZ" }, "source": [ "# Retrieval-Augmented Generation (RAG)" ] }, { "cell_type": "markdown", "metadata": { "id": "CorBhMaiZwdb" }, "source": [ "Install the Hugging Face libraries to run this notebook." ] }, { "cell_type": "code", "execution_count": 98, "metadata": { "id": "CPGVVOEHZwdb", "scrolled": false }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Defaulting to user installation because normal site-packages is not writeable\n", "Requirement already satisfied: transformers in /home/nathanael/.local/lib/python3.10/site-packages (4.48.1)\n", "Requirement already satisfied: wikipedia in /home/nathanael/.local/lib/python3.10/site-packages (1.4.0)\n", "Requirement already satisfied: filelock in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (3.13.1)\n", "Requirement already satisfied: huggingface-hub<1.0,>=0.24.0 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (0.26.2)\n", "Requirement already satisfied: numpy>=1.17 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (1.26.4)\n", "Requirement already satisfied: packaging>=20.0 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (23.2)\n", "Requirement already satisfied: pyyaml>=5.1 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (6.0.1)\n", "Requirement already satisfied: regex!=2019.12.17 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (2023.3.23)\n", "Requirement already satisfied: requests in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (2.32.3)\n", "Requirement already satisfied: tokenizers<0.22,>=0.21 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (0.21.0)\n", "Requirement already satisfied: safetensors>=0.4.1 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (0.4.3)\n", "Requirement already satisfied: tqdm>=4.27 in /home/nathanael/.local/lib/python3.10/site-packages (from transformers) (4.67.0)\n", "Requirement already satisfied: beautifulsoup4 in /home/nathanael/.local/lib/python3.10/site-packages (from wikipedia) (4.12.3)\n", "Requirement already satisfied: fsspec>=2023.5.0 in /home/nathanael/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (2024.2.0)\n", "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/nathanael/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.24.0->transformers) (4.12.2)\n", "Requirement already satisfied: charset-normalizer<4,>=2 in /home/nathanael/.local/lib/python3.10/site-packages (from requests->transformers) (3.3.2)\n", "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (3.3)\n", "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/nathanael/.local/lib/python3.10/site-packages (from requests->transformers) (2.3.0)\n", "Requirement already satisfied: certifi>=2017.4.17 in /home/nathanael/.local/lib/python3.10/site-packages (from requests->transformers) (2024.2.2)\n", "Requirement already satisfied: soupsieve>1.2 in /usr/lib/python3/dist-packages (from beautifulsoup4->wikipedia) (2.3.1)\n" ] } ], "source": [ "!pip install transformers wikipedia" ] }, { "cell_type": "code", "execution_count": 99, "metadata": {}, "outputs": [], "source": [ "import torch\n", "import torch.nn.functional as F\n", "\n", "import wikipedia\n", "import json" ] }, { "cell_type": "code", "execution_count": 100, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'cpu'" ] }, "execution_count": 100, "metadata": {}, "output_type": "execute_result" } ], "source": [ "device = \"cuda\" if torch.cuda.is_available() else \"cpu\"\n", "device" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Document ingestion" ] }, { "cell_type": "code", "execution_count": 101, "metadata": {}, "outputs": [], "source": [ "def extract_wikipedia_pages(page_titles):\n", " \"\"\"\n", " Extracts Wikipedia pages and stores them in a dictionary.\n", "\n", " Args:\n", " page_titles: A list of Wikipedia page titles to extract.\n", "\n", " Returns:\n", " A dictionary containing the text of each Wikipedia page.\n", " \"\"\"\n", "\n", " page_data = {}\n", " for title in page_titles:\n", " try:\n", " page = wikipedia.page(title)\n", " content = page.content.strip()\n", " content = content.replace(\"\\n\", \"\")\n", " page_data[page.title] = content\n", " except wikipedia.exceptions.PageError:\n", " print(f\"Page '{title}' not found.\")\n", " except wikipedia.exceptions.DisambiguationError as e:\n", " print(f\"Disambiguation error for '{title}': {e.options}\")\n", "\n", " return page_data" ] }, { "cell_type": "code", "execution_count": 102, "metadata": {}, "outputs": [], "source": [ "page_titles = [\n", " \"Roger Apéry\",\n", " \"Owen Willans Richardson\",\n", " \"Otto Sackur\",\n", " \"Ludvig Lorenz\",\n", " \"Klaus von Klitzing\",\n", " \"Henri Victor Regnault\",\n", " \"Erwin Madelung\",\n", " ]\n", "\n", "# Uncomment the next line to scroll through Wikipedia\n", "# wikipedia_data = extract_wikipedia_pages(page_titles)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Save the dictionary using `json.dump()`:" ] }, { "cell_type": "code", "execution_count": 103, "metadata": {}, "outputs": [], "source": [ "# with open('wikipedia_data.json', 'w') as f:\n", "# json.dump(wikipedia_data, f, indent=4)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "Load the dictionary using `json.load()`:" ] }, { "cell_type": "code", "execution_count": 104, "metadata": {}, "outputs": [], "source": [ "with open('wikipedia_data.json', 'r') as f:\n", " wikipedia_data = json.load(f)" ] }, { "cell_type": "code", "execution_count": 105, "metadata": { "scrolled": true }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "3107\n", "3455\n", "1683\n", "1873\n", "1762\n", "3431\n", "1487\n" ] } ], "source": [ "for doc in wikipedia_data:\n", " print(len(wikipedia_data[doc]))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Document pre-processing" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "We load just the tokenizer:" ] }, { "cell_type": "code", "execution_count": 106, "metadata": {}, "outputs": [ { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "7cd9cdae6a504235aaa559506873470a", "version_major": 2, "version_minor": 0 }, "text/plain": [ "tokenizer_config.json: 0%| | 0.00/350 [00:00