Add persisting and loaading of FAISS index

notebooks/2.0-solution-rag-ollama.ipynb

@@ -494,75 +494,54 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 23,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "File renamed from '/default__vector_store.json' to '/vector_store.json'.\n",
+      "Loading storage context\n",
+      "Loading index\n"
+     ]
+    }
+   ],
    "source": [
     "import faiss\n",
     "from llama_index.vector_stores.faiss import FaissVectorStore\n",
     "\n",
     "\n",
-    "# dimensions of text-ada-embedding-002\n",
-    "d = 1024\n",
-    "faiss_index = faiss.IndexFlatL2(d)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "from llama_index.vector_stores.faiss import FaissVectorStore\n",
+    "d = 1024  # BAAI/bge-large-en-v1.5 embedding size\n",
+    "faiss_index = faiss.IndexFlatL2(d)\n",
     "\n",
-    "\n",
-    "vector_store = FaissVectorStore(faiss_index=faiss_index)\n",
-    "storage_context = StorageContext.from_defaults(vector_store=vector_store)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "index = VectorStoreIndex.from_documents(\n",
-    "    documents,\n",
-    "    service_context=service_context,\n",
-    "    storage_context=storage_context\n",
-    ")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 10,
-   "metadata": {},
-   "outputs": [],
-   "source": [
     "persist_dir = \"../storage/faiss/\"\n",
-    "persist_dir = str(Path(persist_dir).resolve())"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "index.storage_context.persist(persist_dir=persist_dir)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "vector_store = FaissVectorStore.from_persist_dir(persist_dir)\n",
-    "storage_context = StorageContext.from_defaults(\n",
-    "    vector_store=vector_store, persist_dir=persist_dir\n",
-    ")\n",
-    "index = load_index_from_storage(storage_context=storage_context, service_context=service_context)"
+    "persist_dir = str(Path(persist_dir).resolve())\n",
+    "\n",
+    "if os.path.exists(persist_dir + '/index_store.json'):  # Load\n",
+    "    old_name = '/default__vector_store.json'\n",
+    "    new_name = '/vector_store.json'\n",
+    "    if os.path.exists(persist_dir + old_name):\n",
+    "        os.rename(persist_dir + old_name, persist_dir + new_name)\n",
+    "        print(f\"File renamed from '{old_name}' to '{new_name}'.\")\n",
+    "    print('Loading storage context')\n",
+    "    vector_store = FaissVectorStore.from_persist_dir(persist_dir)\n",
+    "    storage_context = StorageContext.from_defaults(\n",
+    "        vector_store=vector_store, persist_dir=persist_dir\n",
+    "    )\n",
+    "    print('Loading index')\n",
+    "    index = load_index_from_storage(storage_context, service_context=service_context)\n",
+    "else:  # Create\n",
+    "    vector_store = FaissVectorStore(faiss_index=faiss_index)\n",
+    "    storage_context = StorageContext.from_defaults(vector_store=vector_store)\n",
+    "    print('Creating index')\n",
+    "    index = VectorStoreIndex.from_documents(\n",
+    "        documents,\n",
+    "        service_context=service_context,\n",
+    "        storage_context=storage_context\n",
+    "    )\n",
+    "    print('Persisting index')\n",
+    "    index.storage_context.persist(persist_dir=persist_dir)"
    ]
   },
   {
@@ -574,7 +553,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": 24,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -633,19 +612,22 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 12,
+   "execution_count": 27,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "To create a student club at Innopolis University, you can follow these steps:\n",
+      "To create a student club, you can follow these steps:\n",
       "\n",
-      "1. Go to the campuslife.innopolis.ru/clubs page and click on the \"Create my Club\" button.\n",
-      "2. Once you've created your club, you can start organizing activities and events for its members. Joining a student organization will give you opportunities to learn more about yourself, develop soft skills, gain practical experience, network with other students, and even expand your CV.\n",
+      "1. Visit the campuslife.innopolis.ru/clubs page.\n",
+      "2. Click on the \"Create my Club\" button.\n",
+      "3. Fill out the necessary information about your club, such as its name, purpose, and activities.\n",
+      "4. Provide any additional details or requirements for membership.\n",
+      "5. Submit the form to create your student club.\n",
       "\n",
-      "If you have any questions or need help with the process, you can contact Evgeniia Dyuzhakova, @janedyuzha\\_dyuzha, who is the Student Clubs Advisor at Innopolis University.\n"
+      "For questions or further assistance, you can contact Evgeniia Dyuzhakova at @janedyuzha_dyuzha.\n"
      ]
     }
    ],