pdfFindDownload-githubVersion.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "0da90876-b4d9-417f-8633-f3d0d3e53ddf",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "def download_pdfs(url):\n",
    "    response = requests.get(url)\n",
    "    soup = BeautifulSoup(response.content, \"html.parser\")\n",
    "    #find download buttons, exctract download link from \"onclick\" event/attribute\n",
    "    for index, button in enumerate(soup.find_all(\"button\", {\"onclick\": True})):\n",
    "        #style of the onclick url depends on websites\n",
    "        pdf_url = button[\"onclick\"].split(\"'\")[1]\n",
    "        if not pdf_url.startswith(\"http\"):\n",
    "            pdf_url=\"https://\"+soup.find_all('button')[0]['onclick'].split('//')[1].split('?')[0]\n",
    "        response = requests.get(pdf_url)\n",
    "        #save the responce which is pdf file\n",
    "        #open(r\"path_to_save\"+\"\\\\\"+str(index)+\".pdf\", \"wb\").write(response.content)\n",
    "\n",
    "    #delete .js or etc extemsions from url end\n",
    "    if url.split(\"/\")[-1].find(\".\"):\n",
    "        lastPart=url.split(\"/\")[-1].split(\".\")[0]\n",
    "        url=url.split(lastPart)[0]+lastPart\n",
    "\n",
    "    #find all link in response\n",
    "    for index, link in enumerate(soup.find_all(\"a\")):\n",
    "        try:\n",
    "            #find links with \".pdf\" at the end.\n",
    "            if link.get(\"href\").endswith(\".pdf\"):\n",
    "                pdf_url = link.get(\"href\")\n",
    "                #if link starts with \"http\", we download it; otherwise we will concatenate url + anchor/href\n",
    "                if not pdf_url.startswith(\"http\"):\n",
    "                    #try new url, if fails, try to delete common parts\n",
    "                    if requests.get(url+\"/\"+pdf_url).status_code==404:\n",
    "                        lastPart=url.split(\"/\")[-1].split(\".\")[0]\n",
    "                        url=url.split(lastPart)[0][:-1]\n",
    "                    pdf_url=url+\"/\"+pdf_url\n",
    "                print(pdf_url)\n",
    "                response = requests.get(pdf_url)\n",
    "                print(response.status_code)\n",
    "                #save the responce which is pdf file\n",
    "                #open(r\"path_to_save\"+\"\\\\\"+str(index)+\".pdf\", \"wb\").write(response.content)\n",
    "        except:\n",
    "            pass\n",
    "download_pdfs(\"http://example.com\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cce3c24c-4271-4295-af15-757f41d4bc1c",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}