Nlp_approach_ac.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import string\n",
    "import re\n",
    "\n",
    "from nltk.tokenize import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "from pyspark.ml.clustering import KMeans, KMeansModel\n",
    "from pyspark.ml.feature import CountVectorizer, IDF \n",
    "from pyspark.sql.functions import udf\n",
    "\n",
    "from pyspark.sql import Row\n",
    "\n",
    "PUNCTUATION = set(string.punctuation)\n",
    "STOPWORDS = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.clustering import  KMeansModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from nlp_cl_start import print_cl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "api_f = ['attributes.RestaurantsPriceRange2', 'business_id', 'stars', 'review_count', 'categories']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.sql.types import ArrayType, StringType, BooleanType"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.ml.evaluation import ClusteringEvaluator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def if_restaurant(text):\n",
    "    if text is None:\n",
    "        return False\n",
    "    else:\n",
    "        return 'Restaurants' in text\n",
    "\n",
    "if_rest_udf = udf(if_restaurant, BooleanType())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pyspark as ps\n",
    "spark = (ps.sql.SparkSession.builder\n",
    "        .master(\"local[4]\")\n",
    "        .appName(\"yelp_academic\")\n",
    "        .getOrCreate()\n",
    "        )\n",
    "sc = spark.sparkContext"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#from nlp_cl_start import kmean_counts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "biz = spark.read.json('yelp_dataset/yelp_academic_dataset_business.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "rev = spark.read.json('yelp_dataset/yelp_academic_dataset_review.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad = rev.filter('stars < 3')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Row(business_id='iCQpiavjjPzJ5_3gPD5Ebg', cool=0, date='2011-02-25', funny=0, review_id='x7mDIiDB3jEiPGPHOmDzyw', stars=2, text=\"The pizza was okay. Not the best I've had. I prefer Biaggio's on Flamingo / Fort Apache. The chef there can make a MUCH better NY style pizza. The pizzeria @ Cosmo was over priced for the quality and lack of personality in the food. Biaggio's is a much better pick if youre going for italian - family owned, home made recipes, people that actually CARE if you like their food. You dont get that at a pizzeria in a casino. I dont care what you say...\", useful=0, user_id='msQe1u7Z_XuqjGoqhB0J5g'),\n",
       " Row(business_id='jtQARsP6P-LbkyjbO1qNGg', cool=1, date='2014-10-23', funny=1, review_id='LZp4UX5zK3e-c5ZGSeo3kA', stars=1, text='Terrible. Dry corn bread. Rib tips were all fat and mushy and had no flavor. If you want bbq in this neighborhood go to john mulls roadkill grill. Trust me.', useful=3, user_id='msQe1u7Z_XuqjGoqhB0J5g')]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad.take(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1345953"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[business_id: string, cool: bigint, date: string, funny: bigint, review_id: string, stars: bigint, text: string, useful: bigint, user_id: string]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "rests = biz.filter(if_rest_udf(biz.categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [],
   "source": [
    "#rest_rev = rev.join(rests.select('business_id', 'stars').withColumnRenamed('stars','rating'),'business_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_rest_rev = bad.join(rests.select('business_id','stars').withColumnRenamed('stars','rating'),'business_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "768690"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_rest_rev.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[business_id: string, cool: bigint, date: string, funny: bigint, review_id: string, stars: bigint, text: string, useful: bigint, user_id: string, rating: double]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_rest_rev.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_sample = bad_rest_rev.select('business_id','review_id', 'user_id','stars', 'text', 'rating').sample(False, 0.027, seed =91)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.76 ms, sys: 3.61 ms, total: 6.37 ms\n",
      "Wall time: 29.3 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "20765"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "bad_sample.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[business_id: string, review_id: string, user_id: string, stars: bigint, text: string, rating: double]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bad_sample.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def tokenize(text):\n",
    "    regex = re.compile('<.+?>|[^a-zA-Z]')\n",
    "    clean_txt = regex.sub(' ', text)\n",
    "    tokens = clean_txt.split()\n",
    "    lowercased = [t.lower() for t in tokens]\n",
    "\n",
    "    no_punctuation = []\n",
    "    for word in lowercased:\n",
    "        punct_removed = ''.join([letter for letter in word if not letter in PUNCTUATION])\n",
    "        no_punctuation.append(punct_removed)\n",
    "    no_stopwords = [w for w in no_punctuation if not w in STOPWORDS]\n",
    "\n",
    "    STEMMER = PorterStemmer()\n",
    "    stemmed = [STEMMER.stem(w) for w in no_stopwords]\n",
    "    return [w for w in stemmed if w]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "udf_tokenize = udf(f=tokenize, returnType=ArrayType(StringType()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "bad_sample = bad_sample.withColumn('token', udf_tokenize('text'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "cv = CountVectorizer(minDF=10, vocabSize=5000, inputCol='token', outputCol='vectors')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "splits = bad_sample.randomSplit([0.8,0.1,0.1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_k = splits[0]\n",
    "train_ad = splits[1]\n",
    "valid = splits[2]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = cv.fit(train_k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 2.61 ms, sys: 1.74 ms, total: 4.36 ms\n",
      "Wall time: 157 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "sample_vect = model.transform(train_k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save('countvect')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 15 ms, sys: 10.9 ms, total: 25.9 ms\n",
      "Wall time: 1.81 s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>review_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>rating</th>\n",
       "      <th>token</th>\n",
       "      <th>vectors</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-BxWyEIQ6wypT-37MzZizQ</td>\n",
       "      <td>fLFxlSggbp9JJyL11Kt5mw</td>\n",
       "      <td>WBDenhIG-zB271HN9CVAAQ</td>\n",
       "      <td>2</td>\n",
       "      <td>see the waitress?  see her ignore you?  see a ...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>[see, waitress, see, ignor, see, bartend, good...</td>\n",
       "      <td>(0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 3.0, 0.0, 0.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-Miw03v5yXJWjH9MN1aglw</td>\n",
       "      <td>p_zxYEhzM43o52NeFlKunA</td>\n",
       "      <td>Xo99Z-shvqVAijcrQjr_Sg</td>\n",
       "      <td>2</td>\n",
       "      <td>Not a stellar start for Michael Noble.  As a r...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>[stellar, start, michael, nobl, regular, notab...</td>\n",
       "      <td>(3.0, 4.0, 0.0, 0.0, 1.0, 1.0, 2.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-MuatiMmslPOvk9kOMyjkA</td>\n",
       "      <td>iHA6N96oIVQOTJ5biZmhXw</td>\n",
       "      <td>9wF_E3anRNOTq2BrgUXuVg</td>\n",
       "      <td>1</td>\n",
       "      <td>Stopped in for dinner and a glass of wine........</td>\n",
       "      <td>4.0</td>\n",
       "      <td>[stop, dinner, glass, wine, sat, booth, bar, w...</td>\n",
       "      <td>(1.0, 1.0, 1.0, 3.0, 0.0, 1.0, 0.0, 1.0, 1.0, ...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id               review_id                 user_id  \\\n",
       "0  -BxWyEIQ6wypT-37MzZizQ  fLFxlSggbp9JJyL11Kt5mw  WBDenhIG-zB271HN9CVAAQ   \n",
       "1  -Miw03v5yXJWjH9MN1aglw  p_zxYEhzM43o52NeFlKunA  Xo99Z-shvqVAijcrQjr_Sg   \n",
       "2  -MuatiMmslPOvk9kOMyjkA  iHA6N96oIVQOTJ5biZmhXw  9wF_E3anRNOTq2BrgUXuVg   \n",
       "\n",
       "   stars                                               text  rating  \\\n",
       "0      2  see the waitress?  see her ignore you?  see a ...     4.0   \n",
       "1      2  Not a stellar start for Michael Noble.  As a r...     4.0   \n",
       "2      1  Stopped in for dinner and a glass of wine........     4.0   \n",
       "\n",
       "                                               token  \\\n",
       "0  [see, waitress, see, ignor, see, bartend, good...   \n",
       "1  [stellar, start, michael, nobl, regular, notab...   \n",
       "2  [stop, dinner, glass, wine, sat, booth, bar, w...   \n",
       "\n",
       "                                             vectors  \n",
       "0  (0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 3.0, 0.0, 0.0, ...  \n",
       "1  (3.0, 4.0, 0.0, 0.0, 1.0, 1.0, 2.0, 1.0, 1.0, ...  \n",
       "2  (1.0, 1.0, 1.0, 3.0, 0.0, 1.0, 0.0, 1.0, 1.0, ...  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "sample_vect.limit(3).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[business_id: string, review_id: string, user_id: string, stars: bigint, text: string, rating: double, token: array<string>, vectors: vector]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample_vect.cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train k-mean on sample (countvectorizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 222 ms, sys: 58 ms, total: 280 ms\n",
      "Wall time: 20.3 s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cl = 20\n",
    "\n",
    "km = KMeans(k = cl, featuresCol='vectors', maxIter= 30)\n",
    "\n",
    "model_km = km.fit(sample_vect)\n",
    "\n",
    "centers_c = model_km.clusterCenters()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_km.save('kmeans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "mode_kkk = KMeansModel.load('kmeans')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([['get', 'back', 'like', 'time', 'servic', 'good', 'order', 'food',\n",
       "        'go', 'place'],\n",
       "       ['restaur', 'good', 'go', 'time', 'get', 'one', 'place', 'order',\n",
       "        'like', 'food']], dtype='<U15')"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab = np.array(model.vocabulary)\n",
    "vocab[np.argsort(centers_c )[:2,-10:]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['business_id',\n",
       " 'review_id',\n",
       " 'user_id',\n",
       " 'stars',\n",
       " 'text',\n",
       " 'rating',\n",
       " 'token',\n",
       " 'vectors',\n",
       " 'prediction']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pred = model_km.transform(sample_vect)\n",
    "pred.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "ad_vect = model.transform(train_ad)\n",
    "pred_ad = model_km.transform(ad_vect)\n",
    "\n",
    "both = pred_ad.union(pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[review_id: string]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "to_hold = pred_ad.union(pred)\n",
    "to_hold = to_hold.select('review_id')\n",
    "to_hold.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[business_id: string, user_id: string, prediction: int]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "both = both.select('business_id','user_id','prediction')\n",
    "both.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "user_pr = both.select('user_id', 'prediction').withColumnRenamed('prediction','user_cl')\n",
    "biz_pr= both.select('business_id', 'prediction').withColumnRenamed('prediction','biz_cl')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['business_id', 'review_id', 'user_id', 'stars', 'text', 'rating', 'token']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "valid.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2063"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test = valid.select('business_id',  'user_id', 'review_id','stars',  'rating')\n",
    "test.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_join = test.join(user_pr, 'user_id').join(biz_pr, 'business_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_df =test_join.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(108, 107, 108)"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_df['user_id'].unique()), len(test_df['business_id'].unique()), len(test_df['review_id'].unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>rating</th>\n",
       "      <th>user_cl</th>\n",
       "      <th>biz_cl</th>\n",
       "      <th>similar</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>AWJhMTYsIYzCIYAIUA-waA</td>\n",
       "      <td>SPtjWBJmUQOaVManvSLY5Q</td>\n",
       "      <td>5foH5YEx7irPfLa_mOrg8A</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>15</td>\n",
       "      <td>2</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AWJhMTYsIYzCIYAIUA-waA</td>\n",
       "      <td>SPtjWBJmUQOaVManvSLY5Q</td>\n",
       "      <td>5foH5YEx7irPfLa_mOrg8A</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>AWJhMTYsIYzCIYAIUA-waA</td>\n",
       "      <td>SPtjWBJmUQOaVManvSLY5Q</td>\n",
       "      <td>5foH5YEx7irPfLa_mOrg8A</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>AWJhMTYsIYzCIYAIUA-waA</td>\n",
       "      <td>SPtjWBJmUQOaVManvSLY5Q</td>\n",
       "      <td>5foH5YEx7irPfLa_mOrg8A</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>15</td>\n",
       "      <td>1</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AWJhMTYsIYzCIYAIUA-waA</td>\n",
       "      <td>SPtjWBJmUQOaVManvSLY5Q</td>\n",
       "      <td>5foH5YEx7irPfLa_mOrg8A</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id                 user_id               review_id  \\\n",
       "0  AWJhMTYsIYzCIYAIUA-waA  SPtjWBJmUQOaVManvSLY5Q  5foH5YEx7irPfLa_mOrg8A   \n",
       "1  AWJhMTYsIYzCIYAIUA-waA  SPtjWBJmUQOaVManvSLY5Q  5foH5YEx7irPfLa_mOrg8A   \n",
       "2  AWJhMTYsIYzCIYAIUA-waA  SPtjWBJmUQOaVManvSLY5Q  5foH5YEx7irPfLa_mOrg8A   \n",
       "3  AWJhMTYsIYzCIYAIUA-waA  SPtjWBJmUQOaVManvSLY5Q  5foH5YEx7irPfLa_mOrg8A   \n",
       "4  AWJhMTYsIYzCIYAIUA-waA  SPtjWBJmUQOaVManvSLY5Q  5foH5YEx7irPfLa_mOrg8A   \n",
       "\n",
       "   stars  rating  user_cl  biz_cl  similar  \n",
       "0      2     2.5       15       2    False  \n",
       "1      2     2.5       15       0    False  \n",
       "2      2     2.5       15      11    False  \n",
       "3      2     2.5       15       1    False  \n",
       "4      2     2.5       15       0    False  "
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_df['similar']  = (test_df.user_cl == test_df.biz_cl)\n",
    "test_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>similar</th>\n",
       "      <th>stars</th>\n",
       "      <th>rating</th>\n",
       "      <th>pred</th>\n",
       "      <th>act</th>\n",
       "      <th>base</th>\n",
       "      <th>base_3.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1Ct6EYB3JYa9vp092w1JQA</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21VS5dUcc5OtF-sREbgRyw</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2</td>\n",
       "      <td>4.0</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2LuaEfPsNb_77DxPiIIGSg</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2Vn2J41b-wrS__5JcaKrlQ</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32nl-wa3RtRsihVkzB4tVg</th>\n",
       "      <td>2.0</td>\n",
       "      <td>2</td>\n",
       "      <td>3.5</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        similar  stars  rating   pred   act   base  base_3.5\n",
       "review_id                                                                   \n",
       "1Ct6EYB3JYa9vp092w1JQA      2.0      2     3.0   True  True  False      True\n",
       "21VS5dUcc5OtF-sREbgRyw      0.0      2     4.0  False  True  False     False\n",
       "2LuaEfPsNb_77DxPiIIGSg      1.0      2     2.5   True  True   True      True\n",
       "2Vn2J41b-wrS__5JcaKrlQ      0.0      1     3.5  False  True  False     False\n",
       "32nl-wa3RtRsihVkzB4tVg      2.0      2     3.5   True  True  False     False"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_gr_df =  test_df.groupby('review_id').agg({'similar':'sum', 'stars':'mean', 'rating':'mean'})\n",
    "test_gr_df['pred'] = (test_gr_df.similar > 0)\n",
    "test_gr_df['act'] = (test_gr_df.stars < 3)\n",
    "test_gr_df['base'] = (test_gr_df.rating < 3)\n",
    "test_gr_df['base_3.5'] = (test_gr_df.rating < 3.5)\n",
    "\n",
    "\n",
    "test_gr_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_metr import transform_to_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_score, recall_score"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compare accuracy, precision, recall with base model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.37962962962962965, 0.19444444444444445, 0.4537037037037037)"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(test_gr_df.act, test_gr_df.pred), recall_score(test_gr_df.act, test_gr_df.base), recall_score(test_gr_df.act, test_gr_df['base_3.5'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## New test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_T = rev.select('business_id', \n",
    "                   'user_id', 'review_id','stars').join(rests.select('business_id',\n",
    "                                              'stars').withColumnRenamed('stars','rating'),'business_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['business_id', 'user_id', 'review_id', 'stars', 'rating']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_T.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_tf = new_T.join(user_pr, 'user_id').join(biz_pr, 'business_id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "BIG_T = new_tf.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>review_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>rating</th>\n",
       "      <th>user_cl</th>\n",
       "      <th>biz_cl</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-cxD1NimFldATDUsN-oa3A</td>\n",
       "      <td>DeVGAiOf2mHVUDfxvuhVlQ</td>\n",
       "      <td>eZd5Z9HOnR1TeblX81j6xw</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-cxD1NimFldATDUsN-oa3A</td>\n",
       "      <td>Fv0e9RIV9jw5TX3ctA1WbA</td>\n",
       "      <td>TS66wZCd1gF31pSnUlpDFQ</td>\n",
       "      <td>3</td>\n",
       "      <td>2.0</td>\n",
       "      <td>8</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-cxD1NimFldATDUsN-oa3A</td>\n",
       "      <td>qba1JQlp5a6kEwRRTGxdkg</td>\n",
       "      <td>N0jovZTkDJFuU_YF-lKpRQ</td>\n",
       "      <td>2</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-cxD1NimFldATDUsN-oa3A</td>\n",
       "      <td>TqKDnj5_IthGvPCGgN1qvQ</td>\n",
       "      <td>ckLe-X3npxGfXrqLsGm55w</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-cxD1NimFldATDUsN-oa3A</td>\n",
       "      <td>CrEBVkzIQs7tS-wWfaCjNQ</td>\n",
       "      <td>ygg2x40wMrBN-rB05aM8lQ</td>\n",
       "      <td>1</td>\n",
       "      <td>2.0</td>\n",
       "      <td>1</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              business_id                 user_id               review_id  \\\n",
       "0  -cxD1NimFldATDUsN-oa3A  DeVGAiOf2mHVUDfxvuhVlQ  eZd5Z9HOnR1TeblX81j6xw   \n",
       "1  -cxD1NimFldATDUsN-oa3A  Fv0e9RIV9jw5TX3ctA1WbA  TS66wZCd1gF31pSnUlpDFQ   \n",
       "2  -cxD1NimFldATDUsN-oa3A  qba1JQlp5a6kEwRRTGxdkg  N0jovZTkDJFuU_YF-lKpRQ   \n",
       "3  -cxD1NimFldATDUsN-oa3A  TqKDnj5_IthGvPCGgN1qvQ  ckLe-X3npxGfXrqLsGm55w   \n",
       "4  -cxD1NimFldATDUsN-oa3A  CrEBVkzIQs7tS-wWfaCjNQ  ygg2x40wMrBN-rB05aM8lQ   \n",
       "\n",
       "   stars  rating  user_cl  biz_cl  \n",
       "0      3     2.0        0      13  \n",
       "1      3     2.0        8      13  \n",
       "2      2     2.0        0      13  \n",
       "3      1     2.0       13      13  \n",
       "4      1     2.0        1      13  "
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BIG_T.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>similar</th>\n",
       "      <th>stars</th>\n",
       "      <th>rating</th>\n",
       "      <th>pred</th>\n",
       "      <th>act</th>\n",
       "      <th>base</th>\n",
       "      <th>base_3.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>---94vtJ_5o_nikEs6hUjg</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>---L4b6VR6HoB-q7cfMWIA</th>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>---p28WNWGZuG6gLAt-V1w</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>--0y7xOZPpiGD5d6vgfafg</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>--4yS1qzrZI9xoH_2Z-sCA</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        similar  stars  rating   pred    act   base  base_3.5\n",
       "review_id                                                                    \n",
       "---94vtJ_5o_nikEs6hUjg      0.0      5     4.5  False  False  False     False\n",
       "---L4b6VR6HoB-q7cfMWIA      1.0      5     3.5   True  False  False     False\n",
       "---p28WNWGZuG6gLAt-V1w      1.0      2     2.5   True   True   True      True\n",
       "--0y7xOZPpiGD5d6vgfafg      0.0      1     3.5  False   True  False     False\n",
       "--4yS1qzrZI9xoH_2Z-sCA      0.0      4     4.0  False  False  False     False"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BIG_agg = transform_to_score(BIG_T)\n",
    "BIG_agg.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "hold_rev_id = to_hold.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>review_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>iSJkH-ij8BrlvRo_7G3kJA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>47KGVfdMA7y34YUgVJwN2A</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>JJqo-6n87w6yPBSoXmgBOg</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>dbtRpvSLt1vmzeBYb_-FDA</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0XL-4pAarXmZG3IIIqP8ZA</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                review_id\n",
       "0  iSJkH-ij8BrlvRo_7G3kJA\n",
       "1  47KGVfdMA7y34YUgVJwN2A\n",
       "2  JJqo-6n87w6yPBSoXmgBOg\n",
       "3  dbtRpvSLt1vmzeBYb_-FDA\n",
       "4  0XL-4pAarXmZG3IIIqP8ZA"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hold_rev_id.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "BIG_filter = BIG_agg.loc[~BIG_agg.index.isin(hold_rev_id)].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((181996, 7), (181996, 7))"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BIG_agg.shape, BIG_filter.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>similar</th>\n",
       "      <th>stars</th>\n",
       "      <th>rating</th>\n",
       "      <th>pred</th>\n",
       "      <th>act</th>\n",
       "      <th>base</th>\n",
       "      <th>base_3.5</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>review_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>---94vtJ_5o_nikEs6hUjg</th>\n",
       "      <td>0.0</td>\n",
       "      <td>5</td>\n",
       "      <td>4.5</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>---L4b6VR6HoB-q7cfMWIA</th>\n",
       "      <td>1.0</td>\n",
       "      <td>5</td>\n",
       "      <td>3.5</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>---p28WNWGZuG6gLAt-V1w</th>\n",
       "      <td>1.0</td>\n",
       "      <td>2</td>\n",
       "      <td>2.5</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>--0y7xOZPpiGD5d6vgfafg</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1</td>\n",
       "      <td>3.5</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>--4yS1qzrZI9xoH_2Z-sCA</th>\n",
       "      <td>0.0</td>\n",
       "      <td>4</td>\n",
       "      <td>4.0</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        similar  stars  rating   pred    act   base  base_3.5\n",
       "review_id                                                                    \n",
       "---94vtJ_5o_nikEs6hUjg      0.0      5     4.5  False  False  False     False\n",
       "---L4b6VR6HoB-q7cfMWIA      1.0      5     3.5   True  False  False     False\n",
       "---p28WNWGZuG6gLAt-V1w      1.0      2     2.5   True   True   True      True\n",
       "--0y7xOZPpiGD5d6vgfafg      0.0      1     3.5  False   True  False     False\n",
       "--4yS1qzrZI9xoH_2Z-sCA      0.0      4     4.0  False  False  False     False"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "BIG_filter.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "my_pred = BIG_filter.pred\n",
    "act = BIG_filter.act\n",
    "base = BIG_filter.base\n",
    "base_35 = BIG_filter['base_3.5']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6273929097342799, 0.718532275434625, 0.6995813094793292)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(act, my_pred), accuracy_score(act, base), accuracy_score(act, base_35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5912085127945275, 0.17584784103659198, 0.39976112056172863)"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(act, my_pred), recall_score(act, base), recall_score(act, base_35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.5912085127945275, 0.39976112056172863)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recall_score(act, my_pred),  recall_score(act, base_35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(0.6273929097342799, 0.6995813094793292)"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "accuracy_score(act, my_pred),  accuracy_score(act, base_35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "from my_metr import my_scorer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>base</th>\n",
       "      <th>base_3.5</th>\n",
       "      <th>pred</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.718532</td>\n",
       "      <td>0.699581</td>\n",
       "      <td>0.627393</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>recall</th>\n",
       "      <td>0.175848</td>\n",
       "      <td>0.399761</td>\n",
       "      <td>0.591209</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              base  base_3.5      pred\n",
       "accuracy  0.718532  0.699581  0.627393\n",
       "recall    0.175848  0.399761  0.591209"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_scorer(BIG_filter)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "for each row look on bad review restorana, look on bad review usera --- if they come in same cluster --- say 2\n",
    "else say "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "938"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('3DgPtOe-FKRH0bFE3ptzxA', '6IhssoagVtfNMnNjU71Q1A')"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.first()['business_id'], test.first()['user_id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[business_id: string, user_id: string, rating: double, stars: bigint]"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 188,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cluster_rest(business_id, not_user_id):\n",
    "    texts = rest_rev.filter('stars < 3').filter((rest_rev.business_id == business_id)\n",
    "                                        &(rest_rev.user_id!=not_user_id)).select('text')\n",
    "    texts = texts.withColumn('token', udf_tokenize('text'))\n",
    "    vect = model.transform(texts)\n",
    "    #vect.cache()\n",
    "    pred = model_km.transform(vect).select('prediction')\n",
    "    return pred\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 10.4 ms, sys: 3.64 ms, total: 14 ms\n",
      "Wall time: 938 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cl_r = cluster_rest('3DgPtOe-FKRH0bFE3ptzxA','6IhssoagVtfNMnNjU71Q1A')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 187,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cluster_user(not_business_id, user_id):\n",
    "    texts = rest_rev.filter('stars < 3').filter((rest_rev.user_id == user_id)\n",
    "                                        &(rest_rev.business_id!=not_business_id)).select('text')\n",
    "    texts = texts.withColumn('token', udf_tokenize('text'))\n",
    "    vect = model.transform(texts)\n",
    "    #vect.cache()\n",
    "    pred = model_km.transform(vect).select('prediction')\n",
    "    return pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 14.1 ms, sys: 4.93 ms, total: 19 ms\n",
      "Wall time: 719 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "cl_u =cluster_user('3DgPtOe-FKRH0bFE3ptzxA','6IhssoagVtfNMnNjU71Q1A')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 35 ms, sys: 16.8 ms, total: 51.8 ms\n",
      "Wall time: 1.4 s\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "%%time\n",
    "cl_r.intersect(cl_u).count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [],
   "source": [
    "def predict_bad(row):\n",
    "    business_id = row['business_id']\n",
    "    user_id = row['user_id']\n",
    "    mean_rating = row['rating']\n",
    "    cl_u = cluster_user(business_id, user_id)\n",
    "    cl_r = cluster_rest(business_id, user_id)\n",
    "    if cl_r.intersect(cl_u).count() > 0:\n",
    "        return 2\n",
    "    else:\n",
    "        return mean_rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "test1_df = test.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [],
   "source": [
    "#"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [],
   "source": [
    "ttt = test1_df[:4].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 174,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 262 ms, sys: 104 ms, total: 365 ms\n",
      "Wall time: 3min 11s\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "y_h = ttt.apply(predict_bad, axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 175,
   "metadata": {},
   "outputs": [],
   "source": [
    "test2 = rest_rev.select('business_id','user_id','rating', 'stars').sample(False, 0.0027, seed =91)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 176,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[business_id: string, user_id: string, rating: double, stars: bigint]"
      ]
     },
     "execution_count": 176,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test2.cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 177,
   "metadata": {},
   "outputs": [],
   "source": [
    "t2_df = test2.toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t2_df['y_hat'] = t2_df.apply(predict_bad, axis =1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['business_id', 'review_id', 'stars', 'text', 'token', 'vectors', 'prediction']"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Silhouette with squared euclidean distance = -0.1117817836165352\n"
     ]
    }
   ],
   "source": [
    "evaluator = ClusteringEvaluator(featuresCol='vectors')\n",
    "\n",
    "silhouette = evaluator.evaluate(pred)\n",
    "print(\"Silhouette with squared euclidean distance = \" + str(silhouette))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [],
   "source": [
    "count_tr_df =pred.join(biz.select(api_f ).withColumnRenamed('stars', 'rating'), 'business_id').toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19575"
      ]
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(count_tr_df.user_id.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(20765, 12)"
      ]
     },
     "execution_count": 110,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_tr_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>business_id</th>\n",
       "      <th>review_id</th>\n",
       "      <th>user_id</th>\n",
       "      <th>stars</th>\n",
       "      <th>text</th>\n",
       "      <th>token</th>\n",
       "      <th>vectors</th>\n",
       "      <th>RestaurantsPriceRange2</th>\n",
       "      <th>rating</th>\n",
       "      <th>review_count</th>\n",
       "      <th>categories</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>prediction</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "      <td>6718</td>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "      <td>6841</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "      <td>161</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "      <td>3515</td>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "      <td>3550</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "      <td>1619</td>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "      <td>1642</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "      <td>1221</td>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "      <td>1244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "      <td>2399</td>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "      <td>2425</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "      <td>48</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>507</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "      <td>511</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "      <td>216</td>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "      <td>219</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "      <td>354</td>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "      <td>355</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "      <td>556</td>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "      <td>559</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "      <td>293</td>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "      <td>305</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>741</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "      <td>749</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "      <td>1016</td>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "      <td>1026</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "      <td>359</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>55</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>685</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "      <td>691</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>14</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            business_id  review_id  user_id  stars  text  token  vectors  \\\n",
       "prediction                                                                 \n",
       "0                  6841       6841     6841   6841  6841   6841     6841   \n",
       "1                   161        161      161    161   161    161      161   \n",
       "2                  3550       3550     3550   3550  3550   3550     3550   \n",
       "3                  1642       1642     1642   1642  1642   1642     1642   \n",
       "4                  1244       1244     1244   1244  1244   1244     1244   \n",
       "5                  2425       2425     2425   2425  2425   2425     2425   \n",
       "6                    48         48       48     48    48     48       48   \n",
       "7                   511        511      511    511   511    511      511   \n",
       "8                   219        219      219    219   219    219      219   \n",
       "9                   355        355      355    355   355    355      355   \n",
       "10                  559        559      559    559   559    559      559   \n",
       "11                  305        305      305    305   305    305      305   \n",
       "12                  749        749      749    749   749    749      749   \n",
       "13                 1026       1026     1026   1026  1026   1026     1026   \n",
       "14                  359        359      359    359   359    359      359   \n",
       "15                    3          3        3      3     3      3        3   \n",
       "16                    5          5        5      5     5      5        5   \n",
       "17                   56         56       56     56    56     56       56   \n",
       "18                  691        691      691    691   691    691      691   \n",
       "19                   16         16       16     16    16     16       16   \n",
       "\n",
       "            RestaurantsPriceRange2  rating  review_count  categories  \n",
       "prediction                                                            \n",
       "0                             6718    6841          6841        6841  \n",
       "1                              161     161           161         161  \n",
       "2                             3515    3550          3550        3550  \n",
       "3                             1619    1642          1642        1642  \n",
       "4                             1221    1244          1244        1244  \n",
       "5                             2399    2425          2425        2425  \n",
       "6                               48      48            48          48  \n",
       "7                              507     511           511         511  \n",
       "8                              216     219           219         219  \n",
       "9                              354     355           355         355  \n",
       "10                             556     559           559         559  \n",
       "11                             293     305           305         305  \n",
       "12                             741     749           749         749  \n",
       "13                            1016    1026          1026        1026  \n",
       "14                             359     359           359         359  \n",
       "15                               3       3             3           3  \n",
       "16                               5       5             5           5  \n",
       "17                              55      56            56          56  \n",
       "18                             685     691           691         691  \n",
       "19                              14      16            16          16  "
      ]
     },
     "execution_count": 77,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count_tr_df.groupby('prediction').count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## A little more beatiful"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}