data-cleaning-prep.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.7.10","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Anime Recommendation Engine: Data Cleaning & Preparation\n- Reduced input data by 95% to machine learning algorithm by performing analytics on database to extract data that will improve algorithm's performance while minimizing training and inference time\n- Scripts I used to extract relevant data from the csv, transform to JSON and load to machine learning backend of web application\n- Winner at the [Hacktoon](https://devpost.com/software/myanimebutler?ref_content=my-projects-tab&ref_feature=my_projects) hackathon project [MyAnimeButler](https://github.com/arjun-krishna1/MyAnimeButlerData)\n- Inspired by [Collaborative Filtering On Anime Data](https://www.kaggle.com/ajmichelutti/collaborative-filtering-on-anime-data)","metadata":{"_cell_guid":"cde0f61c-f6ab-40b6-a757-c9394566aa62"}},{"cell_type":"code","source":"# Import relevant libraries \n\nimport pandas as pd\nimport numpy as np\nimport scipy as sp\nfrom sklearn.metrics.pairwise import cosine_similarity\nimport operator\nimport json\nimport os\n%matplotlib inline","metadata":{"_cell_guid":"27f2680b-e691-70e4-e762-1bca22449d14","execution":{"iopub.status.busy":"2021-07-20T13:09:25.352749Z","iopub.execute_input":"2021-07-20T13:09:25.353109Z","iopub.status.idle":"2021-07-20T13:09:25.359626Z","shell.execute_reply.started":"2021-07-20T13:09:25.353078Z","shell.execute_reply":"2021-07-20T13:09:25.358557Z"},"trusted":true},"execution_count":30,"outputs":[]},{"cell_type":"code","source":"anime = pd.read_csv('../input/anime.csv')\nrating = pd.read_csv('../input/rating.csv')","metadata":{"_cell_guid":"f5c1d1b7-983a-daa8-27ab-d25011d1f669","execution":{"iopub.status.busy":"2021-07-20T13:09:25.906867Z","iopub.execute_input":"2021-07-20T13:09:25.907417Z","iopub.status.idle":"2021-07-20T13:09:27.383072Z","shell.execute_reply.started":"2021-07-20T13:09:25.907378Z","shell.execute_reply":"2021-07-20T13:09:27.382194Z"},"trusted":true},"execution_count":31,"outputs":[]},{"cell_type":"markdown","source":"# How big is the initial database?","metadata":{}},{"cell_type":"code","source":"print(anime.shape)\nprint(rating.shape)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:09:29.730059Z","iopub.execute_input":"2021-07-20T13:09:29.730555Z","iopub.status.idle":"2021-07-20T13:09:29.735729Z","shell.execute_reply.started":"2021-07-20T13:09:29.730522Z","shell.execute_reply":"2021-07-20T13:09:29.734530Z"},"trusted":true},"execution_count":32,"outputs":[{"name":"stdout","text":"(12294, 7)\n(7813737, 3)\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Replace missing values (-1 in source) with NaN so they don't affect mean","metadata":{}},{"cell_type":"code","source":"rating.rating.replace({-1: np.nan}, regex=True, inplace = True)\nrating.head()","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:09:30.784086Z","iopub.execute_input":"2021-07-20T13:09:30.784751Z","iopub.status.idle":"2021-07-20T13:09:32.581830Z","shell.execute_reply.started":"2021-07-20T13:09:30.784687Z","shell.execute_reply":"2021-07-20T13:09:32.580711Z"},"trusted":true},"execution_count":33,"outputs":[{"execution_count":33,"output_type":"execute_result","data":{"text/plain":"   user_id  anime_id  rating\n0        1        20     NaN\n1        1        24     NaN\n2        1        79     NaN\n3        1       226     NaN\n4        1       241     NaN","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>anime_id</th>\n      <th>rating</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>20</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>1</td>\n      <td>24</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>1</td>\n      <td>79</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>1</td>\n      <td>226</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>1</td>\n      <td>241</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"### We only want TV shows (not comics, etc)","metadata":{}},{"cell_type":"code","source":"anime_tv = anime[anime['type']=='TV']\nanime_tv.head()","metadata":{"_cell_guid":"42760207-0e77-e6cc-03c1-5cb3d2107991","execution":{"iopub.status.busy":"2021-07-20T13:09:32.583524Z","iopub.execute_input":"2021-07-20T13:09:32.583911Z","iopub.status.idle":"2021-07-20T13:09:32.600359Z","shell.execute_reply.started":"2021-07-20T13:09:32.583868Z","shell.execute_reply":"2021-07-20T13:09:32.599334Z"},"trusted":true},"execution_count":34,"outputs":[{"execution_count":34,"output_type":"execute_result","data":{"text/plain":"   anime_id                                               name  \\\n1      5114                   Fullmetal Alchemist: Brotherhood   \n2     28977                                           Gintama°   \n3      9253                                        Steins;Gate   \n4      9969                                      Gintama&#039;   \n5     32935  Haikyuu!!: Karasuno Koukou VS Shiratorizawa Ga...   \n\n                                               genre type episodes  rating  \\\n1  Action, Adventure, Drama, Fantasy, Magic, Mili...   TV       64    9.26   \n2  Action, Comedy, Historical, Parody, Samurai, S...   TV       51    9.25   \n3                                   Sci-Fi, Thriller   TV       24    9.17   \n4  Action, Comedy, Historical, Parody, Samurai, S...   TV       51    9.16   \n5             Comedy, Drama, School, Shounen, Sports   TV       10    9.15   \n\n   members  \n1   793665  \n2   114262  \n3   673572  \n4   151266  \n5    93351  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>anime_id</th>\n      <th>name</th>\n      <th>genre</th>\n      <th>type</th>\n      <th>episodes</th>\n      <th>rating</th>\n      <th>members</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>5114</td>\n      <td>Fullmetal Alchemist: Brotherhood</td>\n      <td>Action, Adventure, Drama, Fantasy, Magic, Mili...</td>\n      <td>TV</td>\n      <td>64</td>\n      <td>9.26</td>\n      <td>793665</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>28977</td>\n      <td>Gintama°</td>\n      <td>Action, Comedy, Historical, Parody, Samurai, S...</td>\n      <td>TV</td>\n      <td>51</td>\n      <td>9.25</td>\n      <td>114262</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>9253</td>\n      <td>Steins;Gate</td>\n      <td>Sci-Fi, Thriller</td>\n      <td>TV</td>\n      <td>24</td>\n      <td>9.17</td>\n      <td>673572</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>9969</td>\n      <td>Gintama&amp;#039;</td>\n      <td>Action, Comedy, Historical, Parody, Samurai, S...</td>\n      <td>TV</td>\n      <td>51</td>\n      <td>9.16</td>\n      <td>151266</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>32935</td>\n      <td>Haikyuu!!: Karasuno Koukou VS Shiratorizawa Ga...</td>\n      <td>Comedy, Drama, School, Shounen, Sports</td>\n      <td>TV</td>\n      <td>10</td>\n      <td>9.15</td>\n      <td>93351</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Join the ratings and anime table to create on database with all information","metadata":{}},{"cell_type":"code","source":"merged = rating.merge(anime_tv, left_on = 'anime_id', right_on = 'anime_id', suffixes= ['_user', ''])\nmerged.rename(columns = {'rating_user':'user_rating'}, inplace = True)\nmerged.head()","metadata":{"_cell_guid":"333be05a-5492-67a2-a64c-154a52ac060f","execution":{"iopub.status.busy":"2021-07-20T13:09:34.357899Z","iopub.execute_input":"2021-07-20T13:09:34.358357Z","iopub.status.idle":"2021-07-20T13:09:35.424334Z","shell.execute_reply.started":"2021-07-20T13:09:34.358328Z","shell.execute_reply":"2021-07-20T13:09:35.423649Z"},"trusted":true},"execution_count":35,"outputs":[{"execution_count":35,"output_type":"execute_result","data":{"text/plain":"   user_id  anime_id  user_rating    name  \\\n0        1        20          NaN  Naruto   \n1        3        20          8.0  Naruto   \n2        5        20          6.0  Naruto   \n3        6        20          NaN  Naruto   \n4       10        20          NaN  Naruto   \n\n                                               genre type episodes  rating  \\\n0  Action, Comedy, Martial Arts, Shounen, Super P...   TV      220    7.81   \n1  Action, Comedy, Martial Arts, Shounen, Super P...   TV      220    7.81   \n2  Action, Comedy, Martial Arts, Shounen, Super P...   TV      220    7.81   \n3  Action, Comedy, Martial Arts, Shounen, Super P...   TV      220    7.81   \n4  Action, Comedy, Martial Arts, Shounen, Super P...   TV      220    7.81   \n\n   members  \n0   683297  \n1   683297  \n2   683297  \n3   683297  \n4   683297  ","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>anime_id</th>\n      <th>user_rating</th>\n      <th>name</th>\n      <th>genre</th>\n      <th>type</th>\n      <th>episodes</th>\n      <th>rating</th>\n      <th>members</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>20</td>\n      <td>NaN</td>\n      <td>Naruto</td>\n      <td>Action, Comedy, Martial Arts, Shounen, Super P...</td>\n      <td>TV</td>\n      <td>220</td>\n      <td>7.81</td>\n      <td>683297</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3</td>\n      <td>20</td>\n      <td>8.0</td>\n      <td>Naruto</td>\n      <td>Action, Comedy, Martial Arts, Shounen, Super P...</td>\n      <td>TV</td>\n      <td>220</td>\n      <td>7.81</td>\n      <td>683297</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>20</td>\n      <td>6.0</td>\n      <td>Naruto</td>\n      <td>Action, Comedy, Martial Arts, Shounen, Super P...</td>\n      <td>TV</td>\n      <td>220</td>\n      <td>7.81</td>\n      <td>683297</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6</td>\n      <td>20</td>\n      <td>NaN</td>\n      <td>Naruto</td>\n      <td>Action, Comedy, Martial Arts, Shounen, Super P...</td>\n      <td>TV</td>\n      <td>220</td>\n      <td>7.81</td>\n      <td>683297</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>10</td>\n      <td>20</td>\n      <td>NaN</td>\n      <td>Naruto</td>\n      <td>Action, Comedy, Martial Arts, Shounen, Super P...</td>\n      <td>TV</td>\n      <td>220</td>\n      <td>7.81</td>\n      <td>683297</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Reduce table size to reduce machine learning training time","metadata":{}},{"cell_type":"code","source":"merged=merged[['user_id', 'anime_id', 'user_rating']]\nmerged_sub= merged[merged.user_id <= 10000]\nmerged_sub.head()","metadata":{"_cell_guid":"15cfaaf3-96c1-b14e-1742-11283997fc7a","execution":{"iopub.status.busy":"2021-07-20T13:09:35.425747Z","iopub.execute_input":"2021-07-20T13:09:35.426020Z","iopub.status.idle":"2021-07-20T13:09:35.656943Z","shell.execute_reply.started":"2021-07-20T13:09:35.425994Z","shell.execute_reply":"2021-07-20T13:09:35.655841Z"},"trusted":true},"execution_count":36,"outputs":[{"execution_count":36,"output_type":"execute_result","data":{"text/plain":"   user_id  anime_id  user_rating\n0        1        20          NaN\n1        3        20          8.0\n2        5        20          6.0\n3        6        20          NaN\n4       10        20          NaN","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>user_id</th>\n      <th>anime_id</th>\n      <th>user_rating</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>1</td>\n      <td>20</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>3</td>\n      <td>20</td>\n      <td>8.0</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>5</td>\n      <td>20</td>\n      <td>6.0</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>6</td>\n      <td>20</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>10</td>\n      <td>20</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"For collaborative filtering we'll need to create a pivot table of users on one axis and tv show names along the other. The pivot table will help us in defining the similarity between users and shows to better predict who will like what.","metadata":{"_cell_guid":"422fb6ee-644a-562f-ae27-99e759feaa6e"}},{"cell_type":"code","source":"piv = merged_sub.pivot_table(index=['user_id'], columns=['anime_id'], values='user_rating')\npiv.head()","metadata":{"_cell_guid":"f7e42fe9-5cf2-c1fb-52ae-1d512c248a72","execution":{"iopub.status.busy":"2021-07-20T13:09:40.076384Z","iopub.execute_input":"2021-07-20T13:09:40.076771Z","iopub.status.idle":"2021-07-20T13:09:40.975833Z","shell.execute_reply.started":"2021-07-20T13:09:40.076737Z","shell.execute_reply":"2021-07-20T13:09:40.974833Z"},"trusted":true},"execution_count":37,"outputs":[{"execution_count":37,"output_type":"execute_result","data":{"text/plain":"anime_id  1      6      7      8      15     16     17     18     19     \\\nuser_id                                                                   \n1           NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n2           NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n3           NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n5           NaN    8.0    NaN    NaN    6.0    NaN    6.0    6.0    NaN   \n7           NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n\nanime_id  20     ...  33028  33037  33046  33113  33222  33241  33274  33341  \\\nuser_id          ...                                                           \n1           NaN  ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n2           NaN  ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n3           8.0  ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n5           6.0  ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    NaN   \n7           NaN  ...    NaN    NaN    NaN    NaN    NaN    NaN    NaN    7.0   \n\nanime_id  33394  33421  \nuser_id                 \n1           NaN    NaN  \n2           NaN    NaN  \n3           NaN    NaN  \n5           NaN    NaN  \n7           NaN    NaN  \n\n[5 rows x 2708 columns]","text/html":"<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th>anime_id</th>\n      <th>1</th>\n      <th>6</th>\n      <th>7</th>\n      <th>8</th>\n      <th>15</th>\n      <th>16</th>\n      <th>17</th>\n      <th>18</th>\n      <th>19</th>\n      <th>20</th>\n      <th>...</th>\n      <th>33028</th>\n      <th>33037</th>\n      <th>33046</th>\n      <th>33113</th>\n      <th>33222</th>\n      <th>33241</th>\n      <th>33274</th>\n      <th>33341</th>\n      <th>33394</th>\n      <th>33421</th>\n    </tr>\n    <tr>\n      <th>user_id</th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n      <th></th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>1</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>8.0</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>5</th>\n      <td>NaN</td>\n      <td>8.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>6.0</td>\n      <td>NaN</td>\n      <td>6.0</td>\n      <td>6.0</td>\n      <td>NaN</td>\n      <td>6.0</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n    <tr>\n      <th>7</th>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>...</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>NaN</td>\n      <td>7.0</td>\n      <td>NaN</td>\n      <td>NaN</td>\n    </tr>\n  </tbody>\n</table>\n<p>5 rows × 2708 columns</p>\n</div>"},"metadata":{}}]},{"cell_type":"markdown","source":"### Turn pivot table to a JSON format\n### Create a list of all the anime's that user has watched, and it's transpose","metadata":{}},{"cell_type":"code","source":"def get_idx_not_nan(row):\n    return list(row[row.notna()].index)\nget_idx_not_nan(piv.iloc[0])","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:09:51.921678Z","iopub.execute_input":"2021-07-20T13:09:51.922009Z","iopub.status.idle":"2021-07-20T13:09:51.929653Z","shell.execute_reply.started":"2021-07-20T13:09:51.921979Z","shell.execute_reply":"2021-07-20T13:09:51.928749Z"},"trusted":true},"execution_count":38,"outputs":[{"execution_count":38,"output_type":"execute_result","data":{"text/plain":"[8074, 11617, 11757, 15451]"},"metadata":{}}]},{"cell_type":"code","source":"def df_to_filter_dict(df, filter_fncn):\n    res = {}\n    for i in range(df.shape[0]):\n        res[i] = filter_fncn(df.iloc[i])\n    return res\n    \n# key is user_id, value is a list of anime_id of anime they reviewed\nuser_to_watched_anime_map = df_to_filter_dict(piv, get_idx_not_nan)\n\n# the output are the animes that this user has reviewed\nprint(user_to_watched_anime_map[0])\nprint(user_to_watched_anime_map[1])","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:09:52.612409Z","iopub.execute_input":"2021-07-20T13:09:52.612804Z","iopub.status.idle":"2021-07-20T13:09:55.999397Z","shell.execute_reply.started":"2021-07-20T13:09:52.612770Z","shell.execute_reply":"2021-07-20T13:09:55.998477Z"},"trusted":true},"execution_count":39,"outputs":[{"name":"stdout","text":"[8074, 11617, 11757, 15451]\n[11771]\n","output_type":"stream"}]},{"cell_type":"code","source":"anime_to_users_that_reviewed_map = df_to_filter_dict(piv.transpose(), get_idx_not_nan)\n\n# the output are the users that have reviewed this anime\nprint(str(anime_to_users_that_reviewed_map[0])[:100] + \"...\")","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:09:56.000689Z","iopub.execute_input":"2021-07-20T13:09:56.000983Z","iopub.status.idle":"2021-07-20T13:09:57.264052Z","shell.execute_reply.started":"2021-07-20T13:09:56.000957Z","shell.execute_reply":"2021-07-20T13:09:57.262888Z"},"trusted":true},"execution_count":40,"outputs":[{"name":"stdout","text":"[19, 21, 23, 32, 34, 43, 46, 50, 51, 55, 68, 72, 80, 81, 103, 126, 129, 139, 152, 160, 163, 173, 175...\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Our efforts reduced the input data to the machine learning algorithms by 99.85%\n### This reduced the training and inference time","metadata":{}},{"cell_type":"code","source":"init_rows = anime.shape[0] + rating.shape[0]\nfinal_rows = len(anime_to_users_that_reviewed_map) + len(user_to_watched_anime_map)\n\nred_percent = round(((init_rows - final_rows) / init_rows)*100, 2)\nprint(f\"Reduction in input data {red_percent}%\")","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:14:46.623766Z","iopub.execute_input":"2021-07-20T13:14:46.624235Z","iopub.status.idle":"2021-07-20T13:14:46.631008Z","shell.execute_reply.started":"2021-07-20T13:14:46.624188Z","shell.execute_reply":"2021-07-20T13:14:46.629550Z"},"trusted":true},"execution_count":48,"outputs":[{"name":"stdout","text":"Reduction in input data 99.85%\n","output_type":"stream"}]},{"cell_type":"markdown","source":"### Output data to JSON format for website backend","metadata":{}},{"cell_type":"code","source":"if not os.path.isdir('data'):\n    os.mkdir(\"data\")\n\nwith open(\"data/users_key_anime_value1.json\", \"w\") as outfile:\n    json.dump(user_to_watched_anime_map, outfile)\nwith open(\"data/anime_key_users_value1.json\", \"w\") as outfile:\n    json.dump(anime_to_users_that_reviewed_map, outfile)","metadata":{"execution":{"iopub.status.busy":"2021-07-20T13:09:14.249677Z","iopub.execute_input":"2021-07-20T13:09:14.250241Z","iopub.status.idle":"2021-07-20T13:09:14.928008Z","shell.execute_reply.started":"2021-07-20T13:09:14.250157Z","shell.execute_reply":"2021-07-20T13:09:14.927192Z"},"trusted":true},"execution_count":29,"outputs":[]}]}