GraphSage-sample-products_gpu0.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT\n",
    "from train_eval import train_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GraphSage on the OGBN-products graph with its original node features. This is to demonstrate the baseline performance of GraphSage on the dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:The OGB package is out of date. Your version is 1.3.1, while the latest version is 1.3.2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00000 | Step 00000 | Loss 8.1067 | Train Acc 0.0107 | Speed (samples/sec) nan | GPU 478.3 MB\n",
      "Epoch 00000 | Step 00020 | Loss 1.3320 | Train Acc 0.6641 | Speed (samples/sec) 9359.7996 | GPU 483.0 MB\n",
      "Epoch 00000 | Step 00040 | Loss 0.9340 | Train Acc 0.7793 | Speed (samples/sec) 9222.7839 | GPU 487.6 MB\n",
      "Epoch 00000 | Step 00060 | Loss 0.8183 | Train Acc 0.8105 | Speed (samples/sec) 9396.8139 | GPU 488.3 MB\n",
      "Epoch 00000 | Step 00080 | Loss 0.6300 | Train Acc 0.8398 | Speed (samples/sec) 9259.6571 | GPU 489.7 MB\n",
      "Epoch 00000 | Step 00100 | Loss 0.5757 | Train Acc 0.8447 | Speed (samples/sec) 9263.0067 | GPU 489.7 MB\n",
      "Epoch 00000 | Step 00120 | Loss 0.5837 | Train Acc 0.8525 | Speed (samples/sec) 9337.0083 | GPU 489.7 MB\n",
      "Epoch 00000 | Step 00140 | Loss 0.6500 | Train Acc 0.8438 | Speed (samples/sec) 9362.1357 | GPU 489.7 MB\n",
      "Epoch 00000 | Step 00160 | Loss 0.5286 | Train Acc 0.8525 | Speed (samples/sec) 9342.7967 | GPU 489.7 MB\n",
      "Epoch 00000 | Step 00180 | Loss 0.5477 | Train Acc 0.8604 | Speed (samples/sec) 9356.3333 | GPU 489.7 MB\n",
      "Epoch Time(s): 24.4550\n",
      "Epoch 00001 | Step 00000 | Loss 0.5178 | Train Acc 0.8516 | Speed (samples/sec) 9283.3396 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00020 | Loss 0.4787 | Train Acc 0.8643 | Speed (samples/sec) 9394.1012 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00040 | Loss 0.5339 | Train Acc 0.8486 | Speed (samples/sec) 9510.7054 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.4806 | Train Acc 0.8691 | Speed (samples/sec) 9602.7381 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.4200 | Train Acc 0.8809 | Speed (samples/sec) 9642.0332 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.4154 | Train Acc 0.8955 | Speed (samples/sec) 9713.4869 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.4671 | Train Acc 0.8838 | Speed (samples/sec) 9731.9768 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.4951 | Train Acc 0.8740 | Speed (samples/sec) 9744.7788 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.5116 | Train Acc 0.8779 | Speed (samples/sec) 9806.2726 | GPU 489.7 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.4708 | Train Acc 0.8867 | Speed (samples/sec) 9829.2861 | GPU 489.7 MB\n",
      "Epoch Time(s): 21.9104\n",
      "Epoch 00002 | Step 00000 | Loss 0.4339 | Train Acc 0.8750 | Speed (samples/sec) 9811.4442 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.4089 | Train Acc 0.8867 | Speed (samples/sec) 9866.3518 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.4757 | Train Acc 0.8789 | Speed (samples/sec) 9910.4015 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.3951 | Train Acc 0.8916 | Speed (samples/sec) 9946.7835 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.4883 | Train Acc 0.8760 | Speed (samples/sec) 10007.9525 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.4765 | Train Acc 0.8760 | Speed (samples/sec) 10039.6515 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.4642 | Train Acc 0.8770 | Speed (samples/sec) 10059.5634 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.3837 | Train Acc 0.9004 | Speed (samples/sec) 10104.1955 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.4513 | Train Acc 0.8779 | Speed (samples/sec) 10152.5723 | GPU 489.7 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.4666 | Train Acc 0.8750 | Speed (samples/sec) 10184.9800 | GPU 489.7 MB\n",
      "Epoch Time(s): 20.9592\n",
      "Epoch 00003 | Step 00000 | Loss 0.4293 | Train Acc 0.8916 | Speed (samples/sec) 10156.1501 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.4637 | Train Acc 0.8584 | Speed (samples/sec) 10166.3342 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.4122 | Train Acc 0.8867 | Speed (samples/sec) 10202.1525 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.4007 | Train Acc 0.9004 | Speed (samples/sec) 10218.8547 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.3735 | Train Acc 0.8906 | Speed (samples/sec) 10254.1437 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.4482 | Train Acc 0.8818 | Speed (samples/sec) 10279.1553 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.4020 | Train Acc 0.8877 | Speed (samples/sec) 10264.5458 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.3802 | Train Acc 0.8945 | Speed (samples/sec) 10277.6254 | GPU 489.7 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.3920 | Train Acc 0.8926 | Speed (samples/sec) 10272.9071 | GPU 489.9 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.3581 | Train Acc 0.8877 | Speed (samples/sec) 10291.1618 | GPU 489.9 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 21.1976\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:23<00:00, 10.25it/s]\n",
      "100%|██████████| 245/245 [00:42<00:00,  5.81it/s]\n",
      "100%|██████████| 245/245 [00:39<00:00,  6.18it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9062, Test Acc: 0.7741\n",
      "Best val acc 0.9062, best test acc: 0.7741\n",
      "Epoch 00004 | Step 00000 | Loss 0.3577 | Train Acc 0.9023 | Speed (samples/sec) 10264.8256 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.3944 | Train Acc 0.8867 | Speed (samples/sec) 10272.3325 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.3535 | Train Acc 0.9014 | Speed (samples/sec) 10289.8488 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.3472 | Train Acc 0.8975 | Speed (samples/sec) 10303.6442 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.3651 | Train Acc 0.8965 | Speed (samples/sec) 10322.4490 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.3265 | Train Acc 0.9199 | Speed (samples/sec) 10348.2410 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.3322 | Train Acc 0.9053 | Speed (samples/sec) 10375.8629 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.3964 | Train Acc 0.8867 | Speed (samples/sec) 10400.8462 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.4405 | Train Acc 0.8926 | Speed (samples/sec) 10409.8531 | GPU 519.3 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.2984 | Train Acc 0.9180 | Speed (samples/sec) 10425.6123 | GPU 519.3 MB\n",
      "Epoch Time(s): 19.9329\n",
      "Epoch 00005 | Step 00000 | Loss 0.3644 | Train Acc 0.9033 | Speed (samples/sec) 10413.6522 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.3220 | Train Acc 0.9092 | Speed (samples/sec) 10422.3182 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.3281 | Train Acc 0.9111 | Speed (samples/sec) 10430.6190 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.3650 | Train Acc 0.9004 | Speed (samples/sec) 10447.6877 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.3234 | Train Acc 0.9072 | Speed (samples/sec) 10473.0302 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.3721 | Train Acc 0.9033 | Speed (samples/sec) 10486.6020 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.3210 | Train Acc 0.9121 | Speed (samples/sec) 10499.5349 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.3527 | Train Acc 0.9023 | Speed (samples/sec) 10514.3083 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.3746 | Train Acc 0.8994 | Speed (samples/sec) 10524.6956 | GPU 519.3 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.3662 | Train Acc 0.8955 | Speed (samples/sec) 10535.5022 | GPU 519.3 MB\n",
      "Epoch Time(s): 19.8815\n",
      "Epoch 00006 | Step 00000 | Loss 0.3949 | Train Acc 0.8955 | Speed (samples/sec) 10535.1704 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.3901 | Train Acc 0.9092 | Speed (samples/sec) 10537.6749 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.3903 | Train Acc 0.8945 | Speed (samples/sec) 10542.3615 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.3749 | Train Acc 0.8848 | Speed (samples/sec) 10549.9506 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.3314 | Train Acc 0.9082 | Speed (samples/sec) 10566.0405 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.3285 | Train Acc 0.9004 | Speed (samples/sec) 10575.0163 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.3256 | Train Acc 0.9092 | Speed (samples/sec) 10576.9735 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.2807 | Train Acc 0.9131 | Speed (samples/sec) 10575.4856 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.3559 | Train Acc 0.8936 | Speed (samples/sec) 10585.1903 | GPU 519.3 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.3361 | Train Acc 0.9014 | Speed (samples/sec) 10592.0007 | GPU 519.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 20.1942\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:23<00:00, 10.48it/s]\n",
      "100%|██████████| 245/245 [00:43<00:00,  5.65it/s]\n",
      "100%|██████████| 245/245 [00:43<00:00,  5.58it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9105, Test Acc: 0.7675\n",
      "Best val acc 0.9105, best test acc: 0.7675\n",
      "Epoch 00007 | Step 00000 | Loss 0.3619 | Train Acc 0.8965 | Speed (samples/sec) 10590.4491 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.4249 | Train Acc 0.8887 | Speed (samples/sec) 10588.9056 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.3932 | Train Acc 0.8955 | Speed (samples/sec) 10583.1057 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.3324 | Train Acc 0.9111 | Speed (samples/sec) 10573.9342 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.3320 | Train Acc 0.9111 | Speed (samples/sec) 10576.0495 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.3143 | Train Acc 0.9033 | Speed (samples/sec) 10574.9021 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.3258 | Train Acc 0.9102 | Speed (samples/sec) 10562.4560 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.3468 | Train Acc 0.9082 | Speed (samples/sec) 10558.0404 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.3935 | Train Acc 0.8936 | Speed (samples/sec) 10550.2988 | GPU 519.3 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.3204 | Train Acc 0.9033 | Speed (samples/sec) 10549.7791 | GPU 519.3 MB\n",
      "Epoch Time(s): 22.1388\n",
      "Epoch 00008 | Step 00000 | Loss 0.3206 | Train Acc 0.9053 | Speed (samples/sec) 10535.1902 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.4005 | Train Acc 0.8701 | Speed (samples/sec) 10529.3720 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.3211 | Train Acc 0.9033 | Speed (samples/sec) 10525.1728 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.3473 | Train Acc 0.9023 | Speed (samples/sec) 10526.4481 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.3861 | Train Acc 0.8916 | Speed (samples/sec) 10534.8521 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.3217 | Train Acc 0.9062 | Speed (samples/sec) 10544.0440 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.3440 | Train Acc 0.9004 | Speed (samples/sec) 10549.6287 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.3478 | Train Acc 0.8975 | Speed (samples/sec) 10550.5494 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.3810 | Train Acc 0.8994 | Speed (samples/sec) 10549.2857 | GPU 519.3 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.3905 | Train Acc 0.9150 | Speed (samples/sec) 10552.5951 | GPU 519.3 MB\n",
      "Epoch Time(s): 21.0307\n",
      "Epoch 00009 | Step 00000 | Loss 0.3453 | Train Acc 0.9092 | Speed (samples/sec) 10544.1031 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.3004 | Train Acc 0.9150 | Speed (samples/sec) 10547.1359 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.3636 | Train Acc 0.8965 | Speed (samples/sec) 10553.4391 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.2995 | Train Acc 0.9033 | Speed (samples/sec) 10556.7143 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.3724 | Train Acc 0.9111 | Speed (samples/sec) 10562.1751 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.3555 | Train Acc 0.9023 | Speed (samples/sec) 10564.0599 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.2947 | Train Acc 0.9189 | Speed (samples/sec) 10565.7888 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.3605 | Train Acc 0.9092 | Speed (samples/sec) 10573.6337 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.3080 | Train Acc 0.9131 | Speed (samples/sec) 10583.0776 | GPU 519.3 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.3208 | Train Acc 0.9131 | Speed (samples/sec) 10585.4131 | GPU 519.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 20.7689\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:23<00:00, 10.40it/s]\n",
      "100%|██████████| 245/245 [00:43<00:00,  5.66it/s]\n",
      "100%|██████████| 245/245 [00:44<00:00,  5.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9155, Test Acc: 0.7772\n",
      "Best val acc 0.9155, best test acc: 0.7772\n",
      "Epoch 00010 | Step 00000 | Loss 0.3008 | Train Acc 0.9170 | Speed (samples/sec) 10576.3577 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.3379 | Train Acc 0.8984 | Speed (samples/sec) 10569.2152 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.2325 | Train Acc 0.9316 | Speed (samples/sec) 10562.1298 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.2980 | Train Acc 0.9121 | Speed (samples/sec) 10555.1847 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.3482 | Train Acc 0.8906 | Speed (samples/sec) 10547.1369 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.3420 | Train Acc 0.9062 | Speed (samples/sec) 10539.3463 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.3384 | Train Acc 0.9102 | Speed (samples/sec) 10539.4163 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.3432 | Train Acc 0.9092 | Speed (samples/sec) 10536.3569 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.3763 | Train Acc 0.9043 | Speed (samples/sec) 10529.0079 | GPU 519.3 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.3686 | Train Acc 0.8994 | Speed (samples/sec) 10513.5498 | GPU 519.3 MB\n",
      "Epoch Time(s): 22.7029\n",
      "Epoch 00011 | Step 00000 | Loss 0.3248 | Train Acc 0.9121 | Speed (samples/sec) 10502.5265 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.3081 | Train Acc 0.9111 | Speed (samples/sec) 10497.0860 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.3254 | Train Acc 0.9072 | Speed (samples/sec) 10489.9300 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.2771 | Train Acc 0.9229 | Speed (samples/sec) 10486.8028 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.3531 | Train Acc 0.9072 | Speed (samples/sec) 10479.3955 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.2942 | Train Acc 0.9111 | Speed (samples/sec) 10480.0570 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.2693 | Train Acc 0.9258 | Speed (samples/sec) 10484.4299 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.3104 | Train Acc 0.9062 | Speed (samples/sec) 10477.8478 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.3261 | Train Acc 0.9189 | Speed (samples/sec) 10473.5033 | GPU 519.3 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.3261 | Train Acc 0.9102 | Speed (samples/sec) 10469.9753 | GPU 519.3 MB\n",
      "Epoch Time(s): 22.4719\n",
      "Epoch 00012 | Step 00000 | Loss 0.3023 | Train Acc 0.9150 | Speed (samples/sec) 10461.5920 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.3655 | Train Acc 0.9053 | Speed (samples/sec) 10463.2049 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.3464 | Train Acc 0.9014 | Speed (samples/sec) 10461.9983 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.3550 | Train Acc 0.8984 | Speed (samples/sec) 10461.6148 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.3110 | Train Acc 0.9248 | Speed (samples/sec) 10460.7619 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.3499 | Train Acc 0.9023 | Speed (samples/sec) 10463.5479 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.3582 | Train Acc 0.8984 | Speed (samples/sec) 10459.8420 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.2992 | Train Acc 0.9111 | Speed (samples/sec) 10459.8251 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.3384 | Train Acc 0.9004 | Speed (samples/sec) 10466.3694 | GPU 519.3 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.3473 | Train Acc 0.9033 | Speed (samples/sec) 10461.9083 | GPU 519.3 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 21.2754\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:24<00:00, 10.13it/s]\n",
      "100%|██████████| 245/245 [00:41<00:00,  5.84it/s]\n",
      "100%|██████████| 245/245 [00:41<00:00,  5.86it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9161, Test Acc: 0.7812\n",
      "Best val acc 0.9161, best test acc: 0.7812\n",
      "Epoch 00013 | Step 00000 | Loss 0.3348 | Train Acc 0.8994 | Speed (samples/sec) 10459.4183 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.4358 | Train Acc 0.9072 | Speed (samples/sec) 10462.9863 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.3222 | Train Acc 0.9102 | Speed (samples/sec) 10461.3774 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.3169 | Train Acc 0.9102 | Speed (samples/sec) 10460.8883 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.3086 | Train Acc 0.9111 | Speed (samples/sec) 10459.5616 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.3139 | Train Acc 0.9131 | Speed (samples/sec) 10458.2342 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.3370 | Train Acc 0.9043 | Speed (samples/sec) 10458.1376 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.3498 | Train Acc 0.8936 | Speed (samples/sec) 10456.6953 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.3684 | Train Acc 0.9014 | Speed (samples/sec) 10451.9349 | GPU 523.0 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.2662 | Train Acc 0.9258 | Speed (samples/sec) 10453.3274 | GPU 523.0 MB\n",
      "Epoch Time(s): 21.4602\n",
      "Epoch 00014 | Step 00000 | Loss 0.3325 | Train Acc 0.9014 | Speed (samples/sec) 10449.3554 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.3462 | Train Acc 0.9082 | Speed (samples/sec) 10451.4241 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.3187 | Train Acc 0.9102 | Speed (samples/sec) 10444.0348 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.3476 | Train Acc 0.8994 | Speed (samples/sec) 10439.9366 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.3606 | Train Acc 0.9062 | Speed (samples/sec) 10437.4110 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.3555 | Train Acc 0.9004 | Speed (samples/sec) 10432.4913 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.3263 | Train Acc 0.9141 | Speed (samples/sec) 10427.9299 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.3500 | Train Acc 0.9043 | Speed (samples/sec) 10425.1813 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.3090 | Train Acc 0.9248 | Speed (samples/sec) 10420.6983 | GPU 523.0 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.2779 | Train Acc 0.9248 | Speed (samples/sec) 10416.1680 | GPU 523.0 MB\n",
      "Epoch Time(s): 22.4713\n",
      "Epoch 00015 | Step 00000 | Loss 0.2788 | Train Acc 0.9170 | Speed (samples/sec) 10403.7629 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.2609 | Train Acc 0.9287 | Speed (samples/sec) 10399.4745 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.3500 | Train Acc 0.8994 | Speed (samples/sec) 10395.4899 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.2798 | Train Acc 0.9131 | Speed (samples/sec) 10393.8155 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.2974 | Train Acc 0.9180 | Speed (samples/sec) 10393.3970 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.3227 | Train Acc 0.9082 | Speed (samples/sec) 10390.5018 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.3004 | Train Acc 0.9102 | Speed (samples/sec) 10390.4866 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.2959 | Train Acc 0.9248 | Speed (samples/sec) 10389.5820 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.3729 | Train Acc 0.9014 | Speed (samples/sec) 10390.4930 | GPU 523.0 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.3250 | Train Acc 0.8994 | Speed (samples/sec) 10389.5913 | GPU 523.0 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 22.2133\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:24<00:00,  9.82it/s]\n",
      "100%|██████████| 245/245 [00:46<00:00,  5.27it/s]\n",
      "100%|██████████| 245/245 [00:45<00:00,  5.34it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9172, Test Acc: 0.7775\n",
      "Best val acc 0.9172, best test acc: 0.7775\n",
      "Epoch 00016 | Step 00000 | Loss 0.2948 | Train Acc 0.9170 | Speed (samples/sec) 10379.6473 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.3186 | Train Acc 0.9111 | Speed (samples/sec) 10381.3923 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.3365 | Train Acc 0.9023 | Speed (samples/sec) 10381.2415 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.3254 | Train Acc 0.9111 | Speed (samples/sec) 10382.6830 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.2897 | Train Acc 0.9102 | Speed (samples/sec) 10380.2726 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.3050 | Train Acc 0.9141 | Speed (samples/sec) 10382.1896 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.3234 | Train Acc 0.9053 | Speed (samples/sec) 10382.4447 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.3076 | Train Acc 0.9111 | Speed (samples/sec) 10382.9582 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.3293 | Train Acc 0.9033 | Speed (samples/sec) 10384.4434 | GPU 524.1 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.2825 | Train Acc 0.9121 | Speed (samples/sec) 10386.2748 | GPU 524.1 MB\n",
      "Epoch Time(s): 21.5217\n",
      "Epoch 00017 | Step 00000 | Loss 0.3532 | Train Acc 0.8984 | Speed (samples/sec) 10379.8449 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.3118 | Train Acc 0.9219 | Speed (samples/sec) 10375.9855 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.3447 | Train Acc 0.9072 | Speed (samples/sec) 10375.3861 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.3495 | Train Acc 0.9053 | Speed (samples/sec) 10378.2353 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.2616 | Train Acc 0.9238 | Speed (samples/sec) 10377.6697 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.3613 | Train Acc 0.8965 | Speed (samples/sec) 10378.1328 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.3151 | Train Acc 0.9258 | Speed (samples/sec) 10379.4343 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.4007 | Train Acc 0.9033 | Speed (samples/sec) 10377.4216 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.2671 | Train Acc 0.9258 | Speed (samples/sec) 10373.7266 | GPU 524.1 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.3812 | Train Acc 0.8896 | Speed (samples/sec) 10376.0123 | GPU 524.1 MB\n",
      "Epoch Time(s): 21.8455\n",
      "Epoch 00018 | Step 00000 | Loss 0.3175 | Train Acc 0.8965 | Speed (samples/sec) 10370.1933 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.3115 | Train Acc 0.9238 | Speed (samples/sec) 10366.2988 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.3052 | Train Acc 0.9131 | Speed (samples/sec) 10364.5206 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.3657 | Train Acc 0.9004 | Speed (samples/sec) 10362.1896 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.2590 | Train Acc 0.9229 | Speed (samples/sec) 10364.6281 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.2759 | Train Acc 0.9209 | Speed (samples/sec) 10365.9961 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.3178 | Train Acc 0.9121 | Speed (samples/sec) 10365.1202 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.3442 | Train Acc 0.9062 | Speed (samples/sec) 10364.7116 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.3177 | Train Acc 0.9189 | Speed (samples/sec) 10367.0213 | GPU 524.1 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.3579 | Train Acc 0.9102 | Speed (samples/sec) 10365.1978 | GPU 524.1 MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/245 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch Time(s): 22.0321\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [00:24<00:00,  9.97it/s]\n",
      "100%|██████████| 245/245 [00:45<00:00,  5.41it/s]\n",
      "100%|██████████| 245/245 [00:44<00:00,  5.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.9201, Test Acc: 0.7832\n",
      "Best val acc 0.9201, best test acc: 0.7832\n",
      "Epoch 00019 | Step 00000 | Loss 0.2961 | Train Acc 0.9121 | Speed (samples/sec) 10360.9258 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.2934 | Train Acc 0.9180 | Speed (samples/sec) 10351.2462 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.3130 | Train Acc 0.9043 | Speed (samples/sec) 10348.1840 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.2941 | Train Acc 0.9043 | Speed (samples/sec) 10341.6833 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.2940 | Train Acc 0.9189 | Speed (samples/sec) 10341.6999 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.3191 | Train Acc 0.9111 | Speed (samples/sec) 10337.9375 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.3202 | Train Acc 0.9072 | Speed (samples/sec) 10334.9447 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.2840 | Train Acc 0.9180 | Speed (samples/sec) 10333.2934 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.2741 | Train Acc 0.9248 | Speed (samples/sec) 10330.9543 | GPU 524.1 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.2938 | Train Acc 0.9229 | Speed (samples/sec) 10331.5492 | GPU 524.1 MB\n",
      "Epoch Time(s): 23.7114\n",
      "Avg epoch time: 27.314659325281777\n",
      "Best val acc 0.9201, best test acc: 0.7832\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "\n",
    "train_g = val_g = test_g = g\n",
    "train_nfeat = val_nfeat = test_nfeat = g.ndata.pop('features')\n",
    "train_labels = val_labels = test_labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 4,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'num_epochs': 20,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:0')\n",
    "\n",
    "# Pack data\n",
    "data = train_g, val_g, test_g, train_nfeat, train_labels, \\\n",
    "           val_nfeat, val_labels, test_nfeat, test_labels\n",
    "# Define model\n",
    "in_feats = train_nfeat.shape[1]\n",
    "model = SAGE(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], F.relu, hyperparams['dropout'])\n",
    "train_sample(model, data, hyperparams, device, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}