Google_scholar+Extract+case+document.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Extraction code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#importing packages\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import pandas as pd\n",
    "import re\n",
    "import os\n",
    "import urllib2\n",
    "import datefinder\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame( columns = [\"cit\", \"Date\"])\n",
    "n = 0\n",
    "for cit_id in citations:\n",
    "    try:\n",
    "        if n%3 == 0:\n",
    "            time.sleep(300)\n",
    "        cit_id1 = \"+\".join(cit_id.split())\n",
    "        url = \"https://scholar.google.com/scholar?hl=en&as_sdt=80006&q=\" + cit_id1 + \"&btnG=\"\n",
    "        r  = requests.get(url)\n",
    "        data = r.text\n",
    "        soup = BeautifulSoup(data)\n",
    "\n",
    "        #to check how many links\n",
    "        r = 0\n",
    "        #finding the releavnt link by Scholar search\n",
    "        for link in soup.find_all('a', href=True):\n",
    "            if \"scholar_case?case\" in str(link['href']):\n",
    "                y = str(link['href'])\n",
    "                y1 = \"https:/\" + y\n",
    "                r = r + 1\n",
    "        case_name = y.split(\"=\")[1]\n",
    "        link_name= \"https://scholar.google.com/scholar_case?case=\" + case_name + \"&q=\" + cit_id1 + \"&hl=en&as_sdt=80006\"\n",
    "\n",
    "        #Opening that link (above) and extract text data  \n",
    "        r1  = requests.get(link_name)\n",
    "        data1 = r1.text\n",
    "        soup1 = BeautifulSoup(data1)\n",
    "        texts  = soup1.findAll(text = True)\n",
    "        ls = \"\".join(texts).split(\"How cited\")[1]\n",
    "\n",
    "\n",
    "        #finding dates\n",
    "        #date_ls = ls.split(\"Signed\")[1]\n",
    "        #matches = list(datefinder.find_dates(date_ls))\n",
    "        #if len(matches) > 0:\n",
    "        # date returned will be a datetime.datetime object. here we are only using the first match.\n",
    "            #date = matches[0]\n",
    "        #else:\n",
    "            #date = 'No dates found'\n",
    "\n",
    "        #Saving in text file\n",
    "        f = open(\"C:\\\\Users\\\\acer1\\\\Desktop\\\\Cit_id_check1\\\\%s.txt\"%cit_id, \"w\")\n",
    "        f.write(ls.encode(\"ascii\", \"ignore\"))\n",
    "        f.close()\n",
    "\n",
    "        #Data for records\n",
    "        cit = cit_id\n",
    "        #Date = date\n",
    "        df.loc[n]= ['cit', 'Date']\n",
    "        n = n + 1\n",
    "        \n",
    "        print n, cit_id\n",
    "    except:\n",
    "        n = n + 1\n",
    "        print \"None\"\n",
    "        pass\n",
    "        "
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "r  = requests.get(\"https://scholar.google.com/scholar_courts?hl=en&as_sdt=0,11\")\n",
    "data = r.text\n",
    "soup = BeautifulSoup(data)`"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "#import numpy as np\n",
    "for i in range(0,298):\n",
    "    y1 = str(soup.find_all(\"span\", class_ = \"gs_in_cb\")[i]).split('value=')[1].split(\"/\")[0]\n",
    "    y2= soup.find_all( class_ = \"gs_in_cb\")[i].text\n",
    "    print y1, y2"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}