sessional 6.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "import numpy as np\n",
    "import math\n",
    "from collections import OrderedDict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"F:\\\\IR Lab\\\\Cranfield\\\\\"\n",
    "\n",
    "file = open(path + \"cran.all.1400\",\"r\").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "filet = re.sub(r'\\n',' ',file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1399"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pattern = re.compile(r'\\.W.+?\\.I')\n",
    "bodytemp =  re.findall(pattern,filet);\n",
    "bodytemp\n",
    "\n",
    "body = []\n",
    "for bd in bodytemp:\n",
    "    body.append(bd[3:-3])\n",
    "    \n",
    "len(body)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "body.append(filet[-668:-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1400"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(body)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize_text(text, min_length = 3, lower = False):\n",
    "    text = re.sub(r'\\s+',\" \",text)\n",
    "    pattern = re.compile(r'^[A-Za-z0-9]*[-]?[A-Za-z0-9]*$')\n",
    "    stopwords = open(\"F:\\\\IR Lab\\\\stopwords.txt\",\"r\", encoding = \"utf-8\").read().split(\"\\n\")\n",
    "    if(lower):\n",
    "        text_sep = [tk.lower() for tk in list(filter(pattern.match,text.split(\" \"))) if len(tk) > min_length and tk.lower() not in stopwords]\n",
    "    else:\n",
    "        text_sep = [tk for tk in list(filter(pattern.match,text.split(\" \"))) if len(tk) > min_length and tk.lower() not in stopwords]\n",
    "    \n",
    "    tk_map = {}\n",
    "    for tk in text_sep:\n",
    "        if(tk_map.get(tk) == None):\n",
    "            tk_map[tk] = 1\n",
    "        else:\n",
    "            tk_map[tk] = tk_map[tk] + 1\n",
    "    \n",
    "    return tk_map"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokens = []\n",
    "for doc in body:\n",
    "    tk = tokenize_text(doc,4,True)\n",
    "    tokens.append(tk)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab = []\n",
    "for tokenlist in tokens:\n",
    "    vocab.extend(tokenlist.keys())\n",
    "\n",
    "vocab = list(set(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "7229"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_mat = []\n",
    "for cterm in vocab:\n",
    "    row = [ tokens[i][cterm] if tokens[i].get(cterm) != None else 0 for i in range(len(tokens))]\n",
    "    tf_mat.append(row)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf_mat = np.array(tf_mat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7229, 1400)"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf_mat.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [],
   "source": [
    "idf = []\n",
    "\n",
    "for cterm in vocab:\n",
    "    count = 0\n",
    "    for tk_map in tokens:\n",
    "        if(tk_map.get(cterm)):\n",
    "            count = count + 1\n",
    "    idf.append(math.log(len(tokens)/count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "# idf = np.reshape(idf, (7262, 1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "idf = np.array(idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7229,)"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "idf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfIdf = []\n",
    "for i in range(len(tf_mat)):\n",
    "    tfIdf.append([ tf*idf[i] for tf in tf_mat[i] ])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "tfIdf = np.array(tfIdf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfIdf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"F:\\\\IR Lab\\\\Cranfield\\\\\"\n",
    "\n",
    "qfile = open(path + \"cran.qry\",\"r\").read()\n",
    "qfilet = re.sub(r'\\n',' ',qfile)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "idf = np.array(idf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "qbodytemp =  re.findall(pattern,qfilet);\n",
    "qbodytemp\n",
    "\n",
    "qbody = []\n",
    "for bd in qbodytemp:\n",
    "    qbody.append(bd[3:-4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [],
   "source": [
    "qbody.append(qfilet[-86:-2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Enter Query number ( from 1 ) : 1\n"
     ]
    }
   ],
   "source": [
    "qnum = int(input(\"Enter Query number ( from 1 ) : \"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "query = qbody[qnum]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {},
   "outputs": [],
   "source": [
    "qterms = tokenize_text(query,4,True)\n",
    "qvector = []\n",
    "for cterm in vocab:\n",
    "    if cterm in qterms:\n",
    "        qvector.append(1)\n",
    "    else:\n",
    "        qvector.append(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 202,
   "metadata": {},
   "outputs": [],
   "source": [
    "qvector = np.array(qvector)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7229,)"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "qvector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Tahera\\AppData\\Local\\Temp\\ipykernel_10488\\1032693783.py:9: RuntimeWarning: invalid value encountered in scalar divide\n",
      "  score = score/(tfIdf.shape[0] - ct)\n"
     ]
    }
   ],
   "source": [
    "cosine = {}\n",
    "for j in range(tfIdf.shape[1]):\n",
    "    score = 0\n",
    "    ct = 0\n",
    "    for i in range(tfIdf.shape[0]):\n",
    "        score = score + tfIdf[i][j]*qvector[i]\n",
    "        if tfIdf[i][j] == 0.0:\n",
    "            ct = ct + 1\n",
    "    score = score/(tfIdf.shape[0] - ct)\n",
    "    cosine[str(j)] = score        \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 205,
   "metadata": {},
   "outputs": [],
   "source": [
    "cs = OrderedDict(sorted(cosine.items(),key=lambda x: x[1], reverse = True))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [],
   "source": [
    "result = list(cs)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'what are the structural and aeroelastic problems associated with flight of high speed aircraft '"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = \"F:\\\\IR Lab\\\\Cranfield\\\\\"\n",
    "file = open(path + \"cranqrel\",\"r\").read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 209,
   "metadata": {},
   "outputs": [],
   "source": [
    "line = file.split(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "ans = {}\n",
    "for i in range(365):\n",
    "    string = str(i+1) + \"\\s\"\n",
    "    pt = re.compile(string)\n",
    "    ele = list(filter(pt.match, line))\n",
    "    ans[str(i+1)] = [str(int(i.split(\" \")[1]) - 1) for i in ele]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 224,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Recall :  0.06896551724137931\n",
      "Precision :  0.2\n"
     ]
    }
   ],
   "source": [
    "R = 0\n",
    "\n",
    "for i in ans[str(qnum)]:\n",
    "    if i in result:\n",
    "        R = R+1\n",
    "  \n",
    "print(\"Recall : \",  R/len(ans[str(qnum)]))\n",
    "print(\"Precision : \",  R/len(result))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}