GAT-sample-eigen-products_gpu3.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using backend: pytorch\n"
     ]
    }
   ],
   "source": [
    "import dgl\n",
    "import time\n",
    "import torch as th\n",
    "import numpy as np\n",
    "import torch.nn.functional as F\n",
    "\n",
    "from load_data import load_ogb\n",
    "from model import SAGE, GAT, get_eigen\n",
    "from train_eval import train_sample"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook shows the performance of GAT on the OGBN-products graph with scaled eigenvectors as positional embeddings for nodes. The positional node embeddings are fixed in this model. This experiment shows that attention-based models cannot use the positional embeddings better than simple neighbor aggregation like GraphSage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:root:The OGB package is out of date. Your version is 1.3.1, while the latest version is 1.3.2.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "load ogbn-products\n",
      "finish loading ogbn-products\n",
      "finish constructing ogbn-products\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:8: UserWarning: Casting complex values to real discards the imaginary part (Triggered internally at  /pytorch/aten/src/ATen/native/Copy.cpp:219.)\n",
      "  \n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/fromnumeric.py:3373: RuntimeWarning: Mean of empty slice.\n",
      "  out=out, **kwargs)\n",
      "/usr/local/lib/python3.6/dist-packages/numpy/core/_methods.py:170: RuntimeWarning: invalid value encountered in double_scalars\n",
      "  ret = ret.dtype.type(ret / rcount)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 00000 | Step 00000 | Loss 3.8491 | Train Acc 0.0371 | Speed (samples/sec) nan | GPU 4325.5 MB\n",
      "Epoch 00000 | Step 00020 | Loss 1.9916 | Train Acc 0.4600 | Speed (samples/sec) 2123.7115 | GPU 4347.8 MB\n",
      "Epoch 00000 | Step 00040 | Loss 1.4747 | Train Acc 0.6094 | Speed (samples/sec) 2220.9872 | GPU 4356.9 MB\n",
      "Epoch 00000 | Step 00060 | Loss 1.2340 | Train Acc 0.6455 | Speed (samples/sec) 2259.4145 | GPU 4356.9 MB\n",
      "Epoch 00000 | Step 00080 | Loss 1.2125 | Train Acc 0.6582 | Speed (samples/sec) 2299.6371 | GPU 4356.9 MB\n",
      "Epoch 00000 | Step 00100 | Loss 1.1147 | Train Acc 0.6787 | Speed (samples/sec) 2300.9666 | GPU 4356.9 MB\n",
      "Epoch 00000 | Step 00120 | Loss 1.0769 | Train Acc 0.6875 | Speed (samples/sec) 2321.6638 | GPU 4356.9 MB\n",
      "Epoch 00000 | Step 00140 | Loss 1.0054 | Train Acc 0.7227 | Speed (samples/sec) 2371.4029 | GPU 4356.9 MB\n",
      "Epoch 00000 | Step 00160 | Loss 0.9208 | Train Acc 0.7441 | Speed (samples/sec) 2412.8443 | GPU 4366.7 MB\n",
      "Epoch 00000 | Step 00180 | Loss 0.8637 | Train Acc 0.7646 | Speed (samples/sec) 2438.5562 | GPU 4366.7 MB\n",
      "Epoch Time(s): 85.5139\n",
      "Epoch 00001 | Step 00000 | Loss 1.0594 | Train Acc 0.7012 | Speed (samples/sec) 2437.3988 | GPU 4366.7 MB\n",
      "Epoch 00001 | Step 00020 | Loss 0.8731 | Train Acc 0.7490 | Speed (samples/sec) 2459.7241 | GPU 4366.7 MB\n",
      "Epoch 00001 | Step 00040 | Loss 0.7882 | Train Acc 0.7676 | Speed (samples/sec) 2484.7543 | GPU 4368.7 MB\n",
      "Epoch 00001 | Step 00060 | Loss 0.7820 | Train Acc 0.7715 | Speed (samples/sec) 2504.1357 | GPU 4384.6 MB\n",
      "Epoch 00001 | Step 00080 | Loss 0.8167 | Train Acc 0.7783 | Speed (samples/sec) 2527.7101 | GPU 4384.6 MB\n",
      "Epoch 00001 | Step 00100 | Loss 0.7449 | Train Acc 0.7881 | Speed (samples/sec) 2553.1117 | GPU 4384.6 MB\n",
      "Epoch 00001 | Step 00120 | Loss 0.8560 | Train Acc 0.7598 | Speed (samples/sec) 2568.6770 | GPU 4384.6 MB\n",
      "Epoch 00001 | Step 00140 | Loss 0.7409 | Train Acc 0.7979 | Speed (samples/sec) 2585.5656 | GPU 4384.6 MB\n",
      "Epoch 00001 | Step 00160 | Loss 0.7395 | Train Acc 0.8027 | Speed (samples/sec) 2600.2612 | GPU 4384.6 MB\n",
      "Epoch 00001 | Step 00180 | Loss 0.7776 | Train Acc 0.7861 | Speed (samples/sec) 2611.1570 | GPU 4384.6 MB\n",
      "Epoch Time(s): 72.2520\n",
      "Epoch 00002 | Step 00000 | Loss 0.8185 | Train Acc 0.7568 | Speed (samples/sec) 2613.3929 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00020 | Loss 0.7209 | Train Acc 0.7871 | Speed (samples/sec) 2623.3424 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00040 | Loss 0.7367 | Train Acc 0.8027 | Speed (samples/sec) 2634.9543 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00060 | Loss 0.7264 | Train Acc 0.7939 | Speed (samples/sec) 2638.9712 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00080 | Loss 0.8055 | Train Acc 0.7803 | Speed (samples/sec) 2641.6763 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00100 | Loss 0.7291 | Train Acc 0.7891 | Speed (samples/sec) 2643.8414 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00120 | Loss 0.7249 | Train Acc 0.7949 | Speed (samples/sec) 2638.8849 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00140 | Loss 0.7218 | Train Acc 0.8008 | Speed (samples/sec) 2636.7799 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00160 | Loss 0.6614 | Train Acc 0.8174 | Speed (samples/sec) 2638.3334 | GPU 4384.6 MB\n",
      "Epoch 00002 | Step 00180 | Loss 0.6892 | Train Acc 0.8135 | Speed (samples/sec) 2639.9714 | GPU 4384.6 MB\n",
      "Epoch Time(s): 75.1653\n",
      "Epoch 00003 | Step 00000 | Loss 0.7908 | Train Acc 0.7783 | Speed (samples/sec) 2636.3038 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00020 | Loss 0.7676 | Train Acc 0.7793 | Speed (samples/sec) 2639.6462 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00040 | Loss 0.7347 | Train Acc 0.7930 | Speed (samples/sec) 2638.2503 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00060 | Loss 0.6911 | Train Acc 0.8066 | Speed (samples/sec) 2638.2356 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00080 | Loss 0.6591 | Train Acc 0.8223 | Speed (samples/sec) 2641.5933 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00100 | Loss 0.6322 | Train Acc 0.8145 | Speed (samples/sec) 2638.4658 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00120 | Loss 0.6607 | Train Acc 0.8223 | Speed (samples/sec) 2637.7897 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00140 | Loss 0.6101 | Train Acc 0.8301 | Speed (samples/sec) 2635.2118 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00160 | Loss 0.6512 | Train Acc 0.8096 | Speed (samples/sec) 2641.2197 | GPU 4384.6 MB\n",
      "Epoch 00003 | Step 00180 | Loss 0.6521 | Train Acc 0.8340 | Speed (samples/sec) 2642.6456 | GPU 4384.6 MB\n",
      "Epoch Time(s): 76.2385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [01:10<00:00,  3.48it/s]\n",
      "100%|██████████| 245/245 [01:49<00:00,  2.24it/s]\n",
      "100%|██████████| 245/245 [01:36<00:00,  2.55it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8339, Test Acc: 0.7096\n",
      "Best val acc 0.8339, best test acc: 0.7096\n",
      "Epoch 00004 | Step 00000 | Loss 0.6511 | Train Acc 0.8223 | Speed (samples/sec) 2638.1725 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00020 | Loss 0.6482 | Train Acc 0.8271 | Speed (samples/sec) 2640.5045 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00040 | Loss 0.5482 | Train Acc 0.8623 | Speed (samples/sec) 2642.3083 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00060 | Loss 0.6082 | Train Acc 0.8223 | Speed (samples/sec) 2643.2295 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00080 | Loss 0.6474 | Train Acc 0.8242 | Speed (samples/sec) 2643.9707 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00100 | Loss 0.5841 | Train Acc 0.8301 | Speed (samples/sec) 2642.9837 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00120 | Loss 0.6487 | Train Acc 0.8252 | Speed (samples/sec) 2648.9848 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00140 | Loss 0.6365 | Train Acc 0.8291 | Speed (samples/sec) 2648.6746 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00160 | Loss 0.5837 | Train Acc 0.8301 | Speed (samples/sec) 2648.2310 | GPU 4384.6 MB\n",
      "Epoch 00004 | Step 00180 | Loss 0.5433 | Train Acc 0.8418 | Speed (samples/sec) 2650.6556 | GPU 4384.6 MB\n",
      "Epoch Time(s): 75.0692\n",
      "Epoch 00005 | Step 00000 | Loss 0.5381 | Train Acc 0.8496 | Speed (samples/sec) 2646.3248 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00020 | Loss 0.5723 | Train Acc 0.8350 | Speed (samples/sec) 2651.0747 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00040 | Loss 0.6619 | Train Acc 0.8271 | Speed (samples/sec) 2646.4524 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00060 | Loss 0.6114 | Train Acc 0.8271 | Speed (samples/sec) 2644.9886 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00080 | Loss 0.6126 | Train Acc 0.8330 | Speed (samples/sec) 2643.6451 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00100 | Loss 0.5645 | Train Acc 0.8477 | Speed (samples/sec) 2646.0117 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00120 | Loss 0.5172 | Train Acc 0.8594 | Speed (samples/sec) 2645.6350 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00140 | Loss 0.5196 | Train Acc 0.8477 | Speed (samples/sec) 2643.7087 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00160 | Loss 0.6147 | Train Acc 0.8320 | Speed (samples/sec) 2642.8651 | GPU 4384.6 MB\n",
      "Epoch 00005 | Step 00180 | Loss 0.6150 | Train Acc 0.8340 | Speed (samples/sec) 2643.6178 | GPU 4384.6 MB\n",
      "Epoch Time(s): 76.8612\n",
      "Epoch 00006 | Step 00000 | Loss 0.6346 | Train Acc 0.7939 | Speed (samples/sec) 2642.5506 | GPU 4384.6 MB\n",
      "Epoch 00006 | Step 00020 | Loss 0.5480 | Train Acc 0.8359 | Speed (samples/sec) 2642.2128 | GPU 4384.6 MB\n",
      "Epoch 00006 | Step 00040 | Loss 0.5561 | Train Acc 0.8486 | Speed (samples/sec) 2641.4491 | GPU 4384.6 MB\n",
      "Epoch 00006 | Step 00060 | Loss 0.6188 | Train Acc 0.8359 | Speed (samples/sec) 2641.0741 | GPU 4384.6 MB\n",
      "Epoch 00006 | Step 00080 | Loss 0.6157 | Train Acc 0.8301 | Speed (samples/sec) 2640.2124 | GPU 4384.6 MB\n",
      "Epoch 00006 | Step 00100 | Loss 0.6034 | Train Acc 0.8320 | Speed (samples/sec) 2644.2529 | GPU 4384.6 MB\n",
      "Epoch 00006 | Step 00120 | Loss 0.5476 | Train Acc 0.8262 | Speed (samples/sec) 2646.0869 | GPU 4384.8 MB\n",
      "Epoch 00006 | Step 00140 | Loss 0.5610 | Train Acc 0.8320 | Speed (samples/sec) 2649.8880 | GPU 4384.8 MB\n",
      "Epoch 00006 | Step 00160 | Loss 0.5423 | Train Acc 0.8418 | Speed (samples/sec) 2652.7369 | GPU 4384.8 MB\n",
      "Epoch 00006 | Step 00180 | Loss 0.5020 | Train Acc 0.8584 | Speed (samples/sec) 2654.5635 | GPU 4384.8 MB\n",
      "Epoch Time(s): 74.0750\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [01:05<00:00,  3.76it/s]\n",
      "100%|██████████| 245/245 [01:49<00:00,  2.23it/s]\n",
      "100%|██████████| 245/245 [01:32<00:00,  2.65it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8389, Test Acc: 0.7047\n",
      "Best val acc 0.8389, best test acc: 0.7047\n",
      "Epoch 00007 | Step 00000 | Loss 0.6828 | Train Acc 0.8145 | Speed (samples/sec) 2653.5629 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00020 | Loss 0.5997 | Train Acc 0.8311 | Speed (samples/sec) 2657.9445 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00040 | Loss 0.6101 | Train Acc 0.8320 | Speed (samples/sec) 2663.3786 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00060 | Loss 0.6190 | Train Acc 0.8320 | Speed (samples/sec) 2667.1244 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00080 | Loss 0.4648 | Train Acc 0.8682 | Speed (samples/sec) 2670.1244 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00100 | Loss 0.5435 | Train Acc 0.8447 | Speed (samples/sec) 2674.2911 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00120 | Loss 0.5693 | Train Acc 0.8379 | Speed (samples/sec) 2674.8970 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00140 | Loss 0.4891 | Train Acc 0.8604 | Speed (samples/sec) 2676.5654 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00160 | Loss 0.6107 | Train Acc 0.8369 | Speed (samples/sec) 2679.6855 | GPU 4384.8 MB\n",
      "Epoch 00007 | Step 00180 | Loss 0.5526 | Train Acc 0.8311 | Speed (samples/sec) 2682.3582 | GPU 4384.8 MB\n",
      "Epoch Time(s): 69.2481\n",
      "Epoch 00008 | Step 00000 | Loss 0.7343 | Train Acc 0.7773 | Speed (samples/sec) 2683.4271 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00020 | Loss 0.5888 | Train Acc 0.8232 | Speed (samples/sec) 2686.6108 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00040 | Loss 0.5499 | Train Acc 0.8564 | Speed (samples/sec) 2689.4923 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00060 | Loss 0.5586 | Train Acc 0.8535 | Speed (samples/sec) 2690.0183 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00080 | Loss 0.5186 | Train Acc 0.8564 | Speed (samples/sec) 2690.4480 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00100 | Loss 0.5463 | Train Acc 0.8369 | Speed (samples/sec) 2693.7604 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00120 | Loss 0.5736 | Train Acc 0.8359 | Speed (samples/sec) 2694.9127 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00140 | Loss 0.5573 | Train Acc 0.8428 | Speed (samples/sec) 2696.4944 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00160 | Loss 0.5389 | Train Acc 0.8506 | Speed (samples/sec) 2697.3238 | GPU 4384.8 MB\n",
      "Epoch 00008 | Step 00180 | Loss 0.5418 | Train Acc 0.8447 | Speed (samples/sec) 2695.4643 | GPU 4384.8 MB\n",
      "Epoch Time(s): 72.2273\n",
      "Epoch 00009 | Step 00000 | Loss 0.5391 | Train Acc 0.8486 | Speed (samples/sec) 2693.9508 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00020 | Loss 0.5586 | Train Acc 0.8428 | Speed (samples/sec) 2692.1502 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00040 | Loss 0.6014 | Train Acc 0.8311 | Speed (samples/sec) 2690.7140 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00060 | Loss 0.5640 | Train Acc 0.8408 | Speed (samples/sec) 2690.5153 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00080 | Loss 0.5384 | Train Acc 0.8457 | Speed (samples/sec) 2689.0723 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00100 | Loss 0.4856 | Train Acc 0.8701 | Speed (samples/sec) 2690.0582 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00120 | Loss 0.5019 | Train Acc 0.8555 | Speed (samples/sec) 2691.2701 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00140 | Loss 0.4528 | Train Acc 0.8652 | Speed (samples/sec) 2691.8138 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00160 | Loss 0.5052 | Train Acc 0.8408 | Speed (samples/sec) 2693.1259 | GPU 4384.8 MB\n",
      "Epoch 00009 | Step 00180 | Loss 0.4927 | Train Acc 0.8574 | Speed (samples/sec) 2695.1448 | GPU 4384.8 MB\n",
      "Epoch Time(s): 74.3819\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [01:06<00:00,  3.68it/s]\n",
      "100%|██████████| 245/245 [01:48<00:00,  2.25it/s]\n",
      "100%|██████████| 245/245 [01:37<00:00,  2.51it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8282, Test Acc: 0.6607\n",
      "Epoch 00010 | Step 00000 | Loss 0.6274 | Train Acc 0.8125 | Speed (samples/sec) 2694.2420 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00020 | Loss 0.5353 | Train Acc 0.8535 | Speed (samples/sec) 2694.8532 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00040 | Loss 0.5478 | Train Acc 0.8428 | Speed (samples/sec) 2696.2151 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00060 | Loss 0.5961 | Train Acc 0.8281 | Speed (samples/sec) 2696.2643 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00080 | Loss 0.4831 | Train Acc 0.8525 | Speed (samples/sec) 2697.5408 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00100 | Loss 0.4956 | Train Acc 0.8506 | Speed (samples/sec) 2697.0649 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00120 | Loss 0.4960 | Train Acc 0.8535 | Speed (samples/sec) 2696.6035 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00140 | Loss 0.4789 | Train Acc 0.8613 | Speed (samples/sec) 2696.3172 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00160 | Loss 0.5285 | Train Acc 0.8418 | Speed (samples/sec) 2694.9499 | GPU 4384.8 MB\n",
      "Epoch 00010 | Step 00180 | Loss 0.5150 | Train Acc 0.8535 | Speed (samples/sec) 2695.1678 | GPU 4384.8 MB\n",
      "Epoch Time(s): 75.1266\n",
      "Epoch 00011 | Step 00000 | Loss 0.6233 | Train Acc 0.8291 | Speed (samples/sec) 2692.9226 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00020 | Loss 0.5676 | Train Acc 0.8535 | Speed (samples/sec) 2693.2831 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00040 | Loss 0.5501 | Train Acc 0.8438 | Speed (samples/sec) 2692.4498 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00060 | Loss 0.4448 | Train Acc 0.8682 | Speed (samples/sec) 2691.9418 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00080 | Loss 0.5312 | Train Acc 0.8350 | Speed (samples/sec) 2692.4963 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00100 | Loss 0.4808 | Train Acc 0.8623 | Speed (samples/sec) 2691.4528 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00120 | Loss 0.4897 | Train Acc 0.8633 | Speed (samples/sec) 2689.8726 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00140 | Loss 0.4988 | Train Acc 0.8623 | Speed (samples/sec) 2689.6896 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00160 | Loss 0.5158 | Train Acc 0.8486 | Speed (samples/sec) 2690.5971 | GPU 4384.8 MB\n",
      "Epoch 00011 | Step 00180 | Loss 0.5435 | Train Acc 0.8555 | Speed (samples/sec) 2690.4852 | GPU 4384.8 MB\n",
      "Epoch Time(s): 76.3548\n",
      "Epoch 00012 | Step 00000 | Loss 0.5539 | Train Acc 0.8506 | Speed (samples/sec) 2688.8772 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00020 | Loss 0.5260 | Train Acc 0.8545 | Speed (samples/sec) 2688.0509 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00040 | Loss 0.5174 | Train Acc 0.8477 | Speed (samples/sec) 2688.4020 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00060 | Loss 0.4858 | Train Acc 0.8574 | Speed (samples/sec) 2687.5842 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00080 | Loss 0.4920 | Train Acc 0.8477 | Speed (samples/sec) 2687.6909 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00100 | Loss 0.5430 | Train Acc 0.8447 | Speed (samples/sec) 2687.7517 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00120 | Loss 0.5156 | Train Acc 0.8408 | Speed (samples/sec) 2688.7883 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00140 | Loss 0.4361 | Train Acc 0.8721 | Speed (samples/sec) 2690.5160 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00160 | Loss 0.4940 | Train Acc 0.8623 | Speed (samples/sec) 2690.1928 | GPU 4384.8 MB\n",
      "Epoch 00012 | Step 00180 | Loss 0.5376 | Train Acc 0.8477 | Speed (samples/sec) 2690.3276 | GPU 4384.8 MB\n",
      "Epoch Time(s): 74.8973\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [01:06<00:00,  3.70it/s]\n",
      "100%|██████████| 245/245 [01:49<00:00,  2.25it/s]\n",
      "100%|██████████| 245/245 [01:41<00:00,  2.41it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8622, Test Acc: 0.7284\n",
      "Best val acc 0.8622, best test acc: 0.7284\n",
      "Epoch 00013 | Step 00000 | Loss 0.5057 | Train Acc 0.8535 | Speed (samples/sec) 2689.4320 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00020 | Loss 0.5321 | Train Acc 0.8447 | Speed (samples/sec) 2688.0146 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00040 | Loss 0.5114 | Train Acc 0.8506 | Speed (samples/sec) 2686.6290 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00060 | Loss 0.5260 | Train Acc 0.8555 | Speed (samples/sec) 2684.7487 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00080 | Loss 0.5228 | Train Acc 0.8555 | Speed (samples/sec) 2680.2325 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00100 | Loss 0.5128 | Train Acc 0.8467 | Speed (samples/sec) 2678.2336 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00120 | Loss 0.4814 | Train Acc 0.8486 | Speed (samples/sec) 2675.8783 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00140 | Loss 0.5047 | Train Acc 0.8574 | Speed (samples/sec) 2674.1120 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00160 | Loss 0.5197 | Train Acc 0.8496 | Speed (samples/sec) 2671.7542 | GPU 4384.8 MB\n",
      "Epoch 00013 | Step 00180 | Loss 0.4468 | Train Acc 0.8652 | Speed (samples/sec) 2669.6956 | GPU 4384.8 MB\n",
      "Epoch Time(s): 84.8802\n",
      "Epoch 00014 | Step 00000 | Loss 0.5453 | Train Acc 0.8379 | Speed (samples/sec) 2667.8490 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00020 | Loss 0.5765 | Train Acc 0.8438 | Speed (samples/sec) 2665.7239 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00040 | Loss 0.5636 | Train Acc 0.8389 | Speed (samples/sec) 2663.4716 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00060 | Loss 0.5423 | Train Acc 0.8477 | Speed (samples/sec) 2660.7850 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00080 | Loss 0.4715 | Train Acc 0.8613 | Speed (samples/sec) 2658.3617 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00100 | Loss 0.5215 | Train Acc 0.8457 | Speed (samples/sec) 2656.4175 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00120 | Loss 0.4575 | Train Acc 0.8652 | Speed (samples/sec) 2654.9807 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00140 | Loss 0.4939 | Train Acc 0.8535 | Speed (samples/sec) 2652.9954 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00160 | Loss 0.4636 | Train Acc 0.8574 | Speed (samples/sec) 2651.1048 | GPU 4384.8 MB\n",
      "Epoch 00014 | Step 00180 | Loss 0.5050 | Train Acc 0.8545 | Speed (samples/sec) 2649.0317 | GPU 4384.8 MB\n",
      "Epoch Time(s): 85.8842\n",
      "Epoch 00015 | Step 00000 | Loss 0.4548 | Train Acc 0.8682 | Speed (samples/sec) 2647.2031 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00020 | Loss 0.5178 | Train Acc 0.8574 | Speed (samples/sec) 2645.6797 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00040 | Loss 0.5171 | Train Acc 0.8525 | Speed (samples/sec) 2644.4175 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00060 | Loss 0.4496 | Train Acc 0.8623 | Speed (samples/sec) 2643.3098 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00080 | Loss 0.4442 | Train Acc 0.8730 | Speed (samples/sec) 2641.6870 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00100 | Loss 0.4864 | Train Acc 0.8672 | Speed (samples/sec) 2641.2142 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00120 | Loss 0.5265 | Train Acc 0.8555 | Speed (samples/sec) 2640.2654 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00140 | Loss 0.4647 | Train Acc 0.8682 | Speed (samples/sec) 2638.9583 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00160 | Loss 0.4826 | Train Acc 0.8555 | Speed (samples/sec) 2638.4417 | GPU 4384.8 MB\n",
      "Epoch 00015 | Step 00180 | Loss 0.5149 | Train Acc 0.8486 | Speed (samples/sec) 2638.1429 | GPU 4384.8 MB\n",
      "Epoch Time(s): 81.1664\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [01:14<00:00,  3.28it/s]\n",
      "100%|██████████| 245/245 [01:58<00:00,  2.07it/s]\n",
      "100%|██████████| 245/245 [01:42<00:00,  2.39it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.8657, Test Acc: 0.7326\n",
      "Best val acc 0.8657, best test acc: 0.7326\n",
      "Epoch 00016 | Step 00000 | Loss 0.5210 | Train Acc 0.8584 | Speed (samples/sec) 2637.3582 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00020 | Loss 0.5073 | Train Acc 0.8535 | Speed (samples/sec) 2637.0436 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00040 | Loss 0.5211 | Train Acc 0.8438 | Speed (samples/sec) 2635.1079 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00060 | Loss 0.3913 | Train Acc 0.8887 | Speed (samples/sec) 2633.9864 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00080 | Loss 0.4725 | Train Acc 0.8750 | Speed (samples/sec) 2633.0194 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00100 | Loss 0.4804 | Train Acc 0.8545 | Speed (samples/sec) 2632.2434 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00120 | Loss 0.4806 | Train Acc 0.8564 | Speed (samples/sec) 2631.6899 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00140 | Loss 0.4811 | Train Acc 0.8555 | Speed (samples/sec) 2630.3185 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00160 | Loss 0.4933 | Train Acc 0.8555 | Speed (samples/sec) 2628.6528 | GPU 4384.8 MB\n",
      "Epoch 00016 | Step 00180 | Loss 0.4575 | Train Acc 0.8594 | Speed (samples/sec) 2627.2312 | GPU 4384.8 MB\n",
      "Epoch Time(s): 82.4667\n",
      "Epoch 00017 | Step 00000 | Loss 0.5225 | Train Acc 0.8486 | Speed (samples/sec) 2625.7929 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00020 | Loss 0.4942 | Train Acc 0.8623 | Speed (samples/sec) 2624.6992 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00040 | Loss 0.4701 | Train Acc 0.8594 | Speed (samples/sec) 2623.5008 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00060 | Loss 0.4918 | Train Acc 0.8545 | Speed (samples/sec) 2622.2934 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00080 | Loss 0.4547 | Train Acc 0.8721 | Speed (samples/sec) 2620.7846 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00100 | Loss 0.4826 | Train Acc 0.8594 | Speed (samples/sec) 2619.5709 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00120 | Loss 0.4558 | Train Acc 0.8711 | Speed (samples/sec) 2618.2580 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00140 | Loss 0.4335 | Train Acc 0.8740 | Speed (samples/sec) 2617.0941 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00160 | Loss 0.4767 | Train Acc 0.8584 | Speed (samples/sec) 2615.9745 | GPU 4384.8 MB\n",
      "Epoch 00017 | Step 00180 | Loss 0.4859 | Train Acc 0.8691 | Speed (samples/sec) 2614.6301 | GPU 4384.8 MB\n",
      "Epoch Time(s): 83.1698\n",
      "Epoch 00018 | Step 00000 | Loss 0.5548 | Train Acc 0.8447 | Speed (samples/sec) 2613.2200 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00020 | Loss 0.4445 | Train Acc 0.8672 | Speed (samples/sec) 2612.3218 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00040 | Loss 0.4630 | Train Acc 0.8691 | Speed (samples/sec) 2611.8018 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00060 | Loss 0.4329 | Train Acc 0.8682 | Speed (samples/sec) 2611.1323 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00080 | Loss 0.5423 | Train Acc 0.8418 | Speed (samples/sec) 2610.2536 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00100 | Loss 0.4442 | Train Acc 0.8711 | Speed (samples/sec) 2609.4666 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00120 | Loss 0.4557 | Train Acc 0.8604 | Speed (samples/sec) 2609.1228 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00140 | Loss 0.5087 | Train Acc 0.8555 | Speed (samples/sec) 2609.1600 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00160 | Loss 0.5337 | Train Acc 0.8516 | Speed (samples/sec) 2608.6304 | GPU 4384.8 MB\n",
      "Epoch 00018 | Step 00180 | Loss 0.4463 | Train Acc 0.8691 | Speed (samples/sec) 2607.9773 | GPU 4384.8 MB\n",
      "Epoch Time(s): 80.3561\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 245/245 [01:14<00:00,  3.29it/s]\n",
      "100%|██████████| 245/245 [01:58<00:00,  2.06it/s]\n",
      "100%|██████████| 245/245 [01:45<00:00,  2.32it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Val acc 0.7670, Test Acc: 0.6219\n",
      "Epoch 00019 | Step 00000 | Loss 0.7854 | Train Acc 0.7764 | Speed (samples/sec) 2606.9743 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00020 | Loss 0.5345 | Train Acc 0.8516 | Speed (samples/sec) 2605.5460 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00040 | Loss 0.4878 | Train Acc 0.8555 | Speed (samples/sec) 2604.1199 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00060 | Loss 0.4896 | Train Acc 0.8594 | Speed (samples/sec) 2602.6144 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00080 | Loss 0.4638 | Train Acc 0.8750 | Speed (samples/sec) 2601.5207 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00100 | Loss 0.4131 | Train Acc 0.8721 | Speed (samples/sec) 2600.2060 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00120 | Loss 0.3935 | Train Acc 0.8789 | Speed (samples/sec) 2599.2342 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00140 | Loss 0.4201 | Train Acc 0.8848 | Speed (samples/sec) 2598.3528 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00160 | Loss 0.4748 | Train Acc 0.8623 | Speed (samples/sec) 2597.7374 | GPU 4384.8 MB\n",
      "Epoch 00019 | Step 00180 | Loss 0.5058 | Train Acc 0.8555 | Speed (samples/sec) 2597.0915 | GPU 4384.8 MB\n",
      "Epoch Time(s): 84.4705\n",
      "Avg epoch time: 98.28608538309733\n",
      "Best val acc 0.8657, best test acc: 0.7326\n"
     ]
    }
   ],
   "source": [
    "g, num_labels = load_ogb('ogbn-products')\n",
    "g.ndata.pop('features')\n",
    "\n",
    "in_feats = 128\n",
    "eigen_vals, eigen_vecs = get_eigen(g, in_feats, 'ogbn-products')\n",
    "train_g = val_g = test_g = g\n",
    "train_nfeat = val_nfeat = test_nfeat = th.tensor(eigen_vecs * np.sqrt(eigen_vals).reshape((1, len(eigen_vals))),\n",
    "                                                 dtype=th.float32)\n",
    "train_labels = val_labels = test_labels = g.ndata.pop('labels')\n",
    "\n",
    "hyperparams = {\n",
    "    'batch_size': 1024,\n",
    "    'num_workers': 0,\n",
    "    'num_hidden': 256,\n",
    "    'num_layers': 3,\n",
    "    'num_heads': 2,\n",
    "    'dropout': 0.5,\n",
    "    'lr': 0.003,\n",
    "    'num_epochs': 20,\n",
    "    'fanouts': [15,10,5],\n",
    "    'eval_batch_size': 10000\n",
    "}\n",
    "device = th.device('cuda:3')\n",
    "\n",
    "# Pack data\n",
    "data = train_g, val_g, test_g, train_nfeat, train_labels, \\\n",
    "           val_nfeat, val_labels, test_nfeat, test_labels\n",
    "# Define model\n",
    "model = GAT(in_feats, hyperparams['num_hidden'], num_labels,\n",
    "             hyperparams['num_layers'], hyperparams['num_heads'], F.relu, hyperparams['dropout'])\n",
    "train_sample(model, data, hyperparams, device, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}