Text Processing_Feature Extraction (1).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import string # special operations on strings\n",
    "import spacy # language models\n",
    "\n",
    "from matplotlib.pyplot import imread\n",
    "from matplotlib import pyplot as plt\n",
    "from wordcloud import WordCloud\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!python -m spacy download en_core_web_md"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas\n",
    "book=pd.read_csv(\"apple.txt\",error_bad_lines=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "book"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "book = [x.strip() for x in book.x] # remove both the leading and the trailing characters\n",
    "book = [x for x in book if x] # removes empty strings, because they are considered in Python as False\n",
    "book[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Joining the list into one string/text\n",
    "text = ' '.join(book)\n",
    "text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Punctuation\n",
    "no_punc_text = text.translate(str.maketrans('', '', string.punctuation)) #with arguments (x, y, z) where 'x' and 'y'\n",
    "# must be equal-length strings and characters in 'x'\n",
    "# are replaced by characters in 'y'. 'z'\n",
    "# is a string (string.punctuation here)\n",
    "no_punc_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Tokenization\n",
    "from nltk.tokenize import word_tokenize\n",
    "text_tokens = word_tokenize(no_punc_text)\n",
    "print(text_tokens[0:50])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(text_tokens)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove stopwords\n",
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "\n",
    "my_stop_words = stopwords.words('english')\n",
    "my_stop_words.append('the')\n",
    "no_stop_tokens = [word for word in text_tokens if not word in my_stop_words]\n",
    "print(no_stop_tokens[0:40])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Noramalize the data\n",
    "lower_words = [x.lower() for x in no_stop_tokens]\n",
    "print(lower_words[0:25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Stemming\n",
    "from nltk.stem import PorterStemmer\n",
    "ps = PorterStemmer()\n",
    "stemmed_tokens = [ps.stem(word) for word in lower_words]\n",
    "print(stemmed_tokens[0:40])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!python -m spacy download en"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# NLP english language model of spacy library\n",
    "nlp = spacy.load('en') "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# lemmas being one of them, but mostly POS, which will follow later\n",
    "doc = nlp(' '.join(no_stop_tokens))\n",
    "print(doc[0:40])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "lemmas = [token.lemma_ for token in doc]\n",
    "print(lemmas[0:25])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Feature Extraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer()\n",
    "X = vectorizer.fit_transform(lemmas)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(vectorizer.vocabulary_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "print(vectorizer.get_feature_names()[50:100])\n",
    "print(X.toarray()[50:100])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(X.toarray().shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Let's see how can bigrams and trigrams can be included here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer_ngram_range = CountVectorizer(analyzer='word',ngram_range=(1,3),max_features = 100)\n",
    "bow_matrix_ngram =vectorizer_ngram_range.fit_transform(book)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(vectorizer_ngram_range.get_feature_names())\n",
    "print(bow_matrix_ngram.toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### TFidf vectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "vectorizer_n_gram_max_features = TfidfVectorizer(norm=\"l2\",analyzer='word', ngram_range=(1,3), max_features = 500)\n",
    "tf_idf_matrix_n_gram_max_features =vectorizer_n_gram_max_features.fit_transform(book)\n",
    "print(vectorizer_n_gram_max_features.get_feature_names())\n",
    "print(tf_idf_matrix_n_gram_max_features.toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Generate wordcloud"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import packages\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from wordcloud import WordCloud, STOPWORDS\n",
    "# Define a function to plot word cloud\n",
    "def plot_cloud(wordcloud):\n",
    "    # Set figure size\n",
    "    plt.figure(figsize=(40, 30))\n",
    "    # Display image\n",
    "    plt.imshow(wordcloud) \n",
    "    # No axis details\n",
    "    plt.axis(\"off\");"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate wordcloud\n",
    "stopwords = STOPWORDS\n",
    "stopwords.add('will')\n",
    "wordcloud = WordCloud(width = 3000, height = 2000, background_color='black', max_words=100,colormap='Set2',stopwords=stopwords).generate(text)\n",
    "# Plot\n",
    "plot_cloud(wordcloud)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save image\n",
    "#wordcloud.to_file(\"wordcloud.png\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}