GD_tensor_rank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "using Zygote\n",
    "using LinearAlgebra\n",
    "using PyPlot"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In this notebook we look into the performance of matrix completion / tensor factorization using neural networks, following [this](http://www.offconvex.org/2021/07/08/imp-reg-tf/) blog post."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train! (generic function with 1 method)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function rand_mat_given_k(n, k)\n",
    "    mat = rand(n,k) * rand(k,n)\n",
    "    return mat ./ norm(mat)\n",
    "end\n",
    "\n",
    "function get_mask(n, p)\n",
    "    mask = [rand()>p for _ in 1:n*n]\n",
    "    return reshape(mask, n, n)\n",
    "end\n",
    "\n",
    "\n",
    "function l2_loss(mat, mask, Ws)\n",
    "    tmp = (*(Ws...) - mat) .* mask\n",
    "    return sum(tmp .^ 2)\n",
    "end\n",
    "\n",
    "function train!(mat, mask, lr, Ws)\n",
    "    _, _, g_Ws = gradient(l2_loss, mat, mask, Ws)\n",
    "    Ws .-= lr .* g_Ws\n",
    "    return nothing\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Matrix completion"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Fixed depth = 2, change learning rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [],
   "source": [
    "m = 2\n",
    "n = 64\n",
    "k = 12\n",
    "p = 0.5\n",
    "n_step = 2000\n",
    "\n",
    "ini_norm = (1e-8).^(1/m)\n",
    "lr = 1e-2\n",
    "\n",
    "Ws = [randn(n,n) for _ in 1:m] .* ini_norm\n",
    "mat = rand_mat_given_k(n, k)\n",
    "# mat = randn(n,n)\n",
    "mask = get_mask(n, p)\n",
    "\n",
    "losses = []\n",
    "errs = []\n",
    "\n",
    "\n",
    "\n",
    "for _ in 1:n_step\n",
    "    train!(mat, mask, lr, Ws)\n",
    "    loss = l2_loss(mat, mask, Ws) ./ sum(mask)\n",
    "    err = l2_loss(mat, .!mask, Ws) ./ sum(.!mask)\n",
    "    push!(losses, loss)\n",
    "    push!(errs, err)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAi8AAAGdCAYAAADaPpOnAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAA9hAAAPYQGoP6dpAABCMUlEQVR4nO3deXxU9b3/8feZyUZCEvaQSAhEQYlAZIkSVEBlEQWl1ooFBVS0tKAi4kJFweWW2nrRe0sE/NWFolZbF6oXKgUF1AoaERQIomAwWEDcSCAsSWbO74/JnMmQBBKY5MycvJ6Px7RnvnNm5nMyGfL2nO9imKZpCgAAIEK47C4AAACgPggvAAAgohBeAABARCG8AACAiEJ4AQAAEYXwAgAAIgrhBQAARBTCCwAAiChRdhcQal6vV7t371ZiYqIMw7C7HAAAUAemaerAgQNKS0uTy3X8cyuOCy+7d+9Wenq63WUAAICTsGvXLnXo0OG4+zguvCQmJkryHXxSUpLN1QAAgLooKSlRenq69Xf8eBwXXvyXipKSkggvAABEmLp0+XBMh928vDxlZWUpJyfH7lIAAEADMpy2qnRJSYmSk5NVXFzMmRcAACJEff5+O+bMCwAAaBoc1+cFAFB3pmmqoqJCHo/H7lLQBERHR8vtdp/y6xBeAKCJKisr0549e3To0CG7S0ETYRiGOnTooObNm5/S6xBeAKAJ8nq9KiwslNvtVlpammJiYpjYEw3KNE199913+uabb9SlS5dTOgNDeAGAJqisrExer1fp6emKj4+3uxw0EW3bttXOnTtVXl5+SuGFDrsA0ISdaBp2IJRCdXaP31oAABBRCC8AACCiEF4AAGhAnTp10hNPPGHdNwxDS5YsabD3W716tQzD0P79+2vdp6FraGh02AUAoBHt2bNHLVu2tLuMiEZ4qaPC70s14n/fVWmZ1+5SbGdIGpyVonPSWwS3G1X3MWppD7RFuVyKchvW/0dXbke7Dbn9bVUei41yKz7GrYTYKMXHuBUfEyW3i6GdAE5NeXm5oqOjG+392rdv32jv5VSElzoqLvxEH+t6xcWW211KeNhReWsAJWa8FnpGqEJuVShK5XLLI7fK5dZhM1alitNhxarUjNMhxeqQGadSxeqQ4lRxzK/01b1P0+hzO6pjq3i1bR4rF2EHqJFpmjpcbs8su82i3XUehTJo0CD17NlTcXFx+vOf/6yYmBhNmjRJs2fPtvYpKirSrbfeqrffflsul0uXXnqp/vSnPyklJUWSNHv2bC1ZskS33XabHnnkEe3cuVMej0cul0sLFizQm2++qXfeeUcZGRl65pln1LZtW02cOFH5+fnq2bOnnn/+eZ1++umSpB07dmjatGlat26dSktL1a1bN82ZM0eDBw+u9RgMw9Drr7+uUaNGafbs2XrwwQer7fPss89qwoQJMk1Tf/zjH7VgwQLt2bNHXbt21f3336+rr77a2nfZsmWaOnWqdu3apX79+mn8+PF1+llWtWnTJt1+++1au3at4uPj9fOf/1xz5861JpNbvXq17r77bm3ZskXR0dE6++yz9eKLLyojI0Offvqppk6dqo8//liGYahLly5auHCh+vbtW+866orwUkepMUfUzEVwaQzJxiHd7frbST232jKjW3y3IrOtrin/tXaZ7bRPLWTKpU6t4/W3X+WqXVLcKdcMRLrD5R5lPbDclvcueGiY4mPq/udo0aJFmjZtmj788EOtXbtWEyZM0Pnnn68hQ4bINE2NGjVKCQkJWrNmjSoqKvSb3/xGo0eP1urVq63X2L59u/72t7/p1VdfDZpv5OGHH9bcuXM1d+5c3XPPPRozZowyMzM1Y8YMdezYUTfeeKOmTJmif/7zn5KkgwcP6rLLLtMjjzyiuLg4LVq0SCNHjtS2bdvUsWPHEx7L9OnTNWnSJOv+Cy+8oAceeMD6wz9z5ky99tprmj9/vrp06aJ3331X1113ndq2bauBAwdq165duuqqqzRp0iT9+te/1scff6w777yzzj9LSTp06JAuvfRS9evXT/n5+dq3b58mTpyoKVOm6LnnnlNFRYVGjRqlm2++WX/9619VVlamjz76yAqcY8eOVa9evTR//ny53W5t3Lixwc9kEV7qKCWzp9SslXT4R7tLsZVp/U/DONUpAGp7fobxnV6JfUhSlYBzUNJ/S2u8PXRj+d2KjY7Wh/cNVmJc450+BlB/PXv21KxZsyRJXbp00bx58/T2229ryJAhWrlypT777DMVFhYqPT1dkrR48WKdffbZys/PV05OjiTfJH2LFy9W27Ztg177hhtu0DXXXCNJuueee5Sbm6v7779fw4YNkyTdfvvtuuGGG6z9s7OzlZ2dbd1/5JFH9Prrr+uNN97QlClTTngszZs3t85urFu3TjNnztSiRYvUvXt3lZaWau7cuXrnnXeUm5srScrMzNT777+vhQsXauDAgZo/f74yMzP1+OOPyzAMnXnmmdq0aZMeffTROv88X3jhBR0+fFh/+ctflJCQIEmaN2+eRo4cqUcffVTR0dEqLi7WiBEjrDNO3bp1s55fVFSku+66S2eddZYk32fS0AgvdZWYIt1TaHcVtmvQiy75T0vLpkvmqfcrqhqyjg00x94f6N6k7a7rfXfmSAVmui4vf1SzRmRpwvmdT7kWIBI0i3ar4KFhJ/Xcl/J36c/vfqWJAzJ1bU76Sb13ffTs2TPofmpqqvbt2ydJ2rp1q9LT063gIklZWVlq0aKFtm7daoWXjIyMasHl2Nf2X2bq0aNHUNuRI0dUUlKipKQklZaW6sEHH9T//d//affu3aqoqNDhw4dVVFRUr2MqKirSqFGjNH36dCs8FRQU6MiRIxoyZEjQvmVlZerVq5d1vP369Qu67OYPOnW1detWZWdnW8FFks4//3x5vV5t27ZNAwYM0IQJEzRs2DANGTJEgwcP1jXXXKPU1FRJ0rRp0zRx4kQtXrxYgwcP1i9+8Qsr5DQUwgvCR85NvlsIWF/jV26SNr8S9FhNwaZqoMkydumrmDHSv6Rlb+XoT60f0D+nDghJXUC4MgyjXpduqrrx/M66sRGD/rGXJAzDkNfr+48e0zRr7D9zbHvVP9S1vbZ//5ra/O931113afny5Xrsscd0xhlnqFmzZrr66qtVVlZW5+MpLS3VFVdcodzcXD300ENWu/89li5dqtNOOy3oObGxsdZxnarafmZS4HifffZZ3XbbbXrrrbf08ssva+bMmVqxYoX69eun2bNna8yYMVq6dKn++c9/atasWXrppZf0s5/97JRrqw3hBc529dO+WxXWV/SpQdLuDZICgebYMDPcla/hP41U8QPN9MvWf9Oy2wkxQDjLyspSUVGRdu3aZZ19KSgoUHFxcdCljlB57733NGHCBOsP9cGDB7Vz5846P980TV133XXyer1avHhxUIjIyspSbGysioqKNHDgwBqfn5WVVW2+lnXr1tXrGLKysrRo0SKVlpZaoe7f//63XC6Xunbtau3Xq1cv9erVSzNmzFBubq5efPFF9evXT5LUtWtXde3aVXfccYd++ctf6tlnn23Q8MIkdWi6blktzS6WZhfLmF0so4ev976pQL8Yw/DdkozDWvrjSO15oJNu++sntpUM4PgGDx6snj17auzYsfrkk0/00Ucfady4cRo4cGCDjH4544wz9Nprr2njxo369NNPNWbMGOuMSV3Mnj1bK1eu1MKFC3Xw4EHt3btXe/fu1eHDh5WYmKjp06frjjvu0KJFi7Rjxw5t2LBBeXl5WrRokSRp0qRJ1oinbdu26cUXX9Rzzz1Xr2MYO3as4uLiNH78eG3evFmrVq3Srbfequuvv14pKSkqLCzUjBkztHbtWn399df617/+pS+++ELdunXT4cOHNWXKFK1evVpff/21/v3vfys/P79BgmJVhBfA7+qng4KMV74QY5qBENPe+En/8/lF2jy7l93VAqiBf+bYli1basCAARo8eLAyMzP18ssvN8j7Pf7442rZsqX69++vkSNHatiwYerdu3edn79mzRodPHhQ/fv3V2pqqnXz1/vwww/rgQce0Jw5c9StWzcNGzZMb775pjp39l2m69ixo1599VW9+eabys7O1oIFC/S73/2uXscQHx+v5cuX68cff1ROTo6uvvpqXXLJJZo3b571+Oeff66f//zn6tq1q2655RZNmTJFv/rVr+R2u/XDDz9o3Lhx6tq1q6655hoNHz68xuHfoWSYobhgFkZKSkqUnJys4uJiJSUl2V0OHODg789SwuE9Qf1iTFP61myplZet0XX9MuwrDjhJR44cUWFhoTp37qy4OKYLQOM43u9dff5+c+YFOIHm934uY8RceeQKupyUYvyksf/sqTW/u8LeAgGgiSG8AHWRc5Pcs38Kupzkv5Q04OgaffzfDdcxDQAQjPAC1MfVT8s1u1j7XG2DzsL0KXlHXzzUx9bSAKCpILwAJyFl1nYVxZ0ZFGC6eLZzBgYAGgHhBThJGTM+0vqki6udgfl6zrn2FgYADueY8JKXl6esrCxr6megMfS98/VqAabjkW2+pQ4AAA3CMeFl8uTJKigoUH5+vt2loInpe+fr+viYAGMunUaAAYAG4pjwAtgp587XtTpmQCDASCpfepetNQGAUxFegBC56L439bnR2QowUaaH/i8A1KlTJz3xxBPWff8swA1l9erVMgxD+/fvb7D3sBvhBQihbrM36kvzNGseGPq/ADjWnj17NHz4cLvLiGiEFyDE1l66zNo2DKl86d02VgPgRMrLyxv1/dq3b6/Y2NhGfc9TVdvPqLF/dn6EFyDExvfvpKVmbpXLRxX6fm5/e4sCHGLQoEG67bbbdPfdd6tVq1Zq3769Zs+eHbRPUVGRrrzySjVv3lxJSUm65ppr9O2331qPz549W+ecc46eeeYZZWZmKjY2VqZpyjAMLVy4UCNGjFB8fLy6deumtWvXavv27Ro0aJASEhKUm5urHTt2WK+1Y8cOXXnllUpJSVHz5s2Vk5OjlStXHvcYql42mj17tgzDqHbzrwxtmqb+8Ic/KDMzU82aNVN2drZeeeWVoNdbtmyZunbtqmbNmumiiy7Szp07T/hzLC4u1i233KJ27dopKSlJF198sT799NM6/YwWLFigK6+8UgkJCXrkkUdO+F4NgfACNID9ly3UdjNNku/sS+viLTZXBJyAaUplpfbc6rk+8KJFi5SQkKAPP/xQf/jDH/TQQw9pxYoVlYdhatSoUfrxxx+1Zs0arVixQjt27NDo0aODXmP79u3629/+pldffVUbN2602h9++GGNGzdOGzdu1FlnnaUxY8boV7/6lWbMmKGPP/5YkjRlyhRr/4MHD+qyyy7TypUrtWHDBg0bNkwjR45UUVFRnY5l+vTp2rNnj3V77LHHFB8fr759+0qSZs6cqWeffVbz58/Xli1bdMcdd+i6667TmjVrJEm7du3SVVddpcsuu0wbN27UxIkTde+99x73PU3T1OWXX669e/dq2bJlWr9+vXr37q1LLrlEP/744wl/RrNmzdKVV16pTZs26cYbb6zTcYZalC3vCjjcdf0ydMsXi7VwxyXWatTfz81Vm2lr7S0MqE35Iel3afa89293SzEJdd69Z8+emjVrliSpS5cumjdvnt5++20NGTJEK1eu1GeffabCwkKlp6dLkhYvXqyzzz5b+fn51lxgZWVlWrx4sdq2bRv02jfccIOuueYaSdI999yj3Nxc3X///Ro2bJgk6fbbb9cNN9xg7Z+dna3s7Gzr/iOPPKLXX39db7zxRlDIqU3z5s3VvHlzSdK6des0c+ZMLVq0SN27d1dpaanmzp2rd955R7m5uZKkzMxMvf/++1q4cKEGDhyo+fPnKzMzU48//rgMw9CZZ56pTZs26dFHH631PVetWqVNmzZp37591uWrxx57TEuWLNErr7yiW2655bg/ozFjxtgWWvw48wI0kKfG9dVyb1+r827r4gI67wIh0LNnz6D7qamp2rdvnyRp69atSk9Pt4KLJGVlZalFixbaunWr1ZaRkVHtj/Kxr52SkiJJ6tGjR1DbkSNHVFJSIkkqLS3V3Xffbb1H8+bN9fnnn9f5zItfUVGRRo0apenTp1vhqaCgQEeOHNGQIUOskNO8eXP95S9/sS5dbd26Vf369ZPh/68kyQo6tVm/fr0OHjyo1q1bB71uYWFh0CWx2n5G/rNCduLMC9CAlnR9VEO2D5Zbpq/z7j9/q+icm+wuC6guOt53BuRkfPIX6YN5Uv8pUu9xJ/fe9dk9OjrovmEY8nq9kmT1yzjWse0JCTWf6an62v79a2rzv99dd92l5cuX67HHHtMZZ5yhZs2a6eqrr1ZZWVmdj6e0tFRXXHGFcnNz9dBDD1nt/vdYunSpTjvttKDn+M+YmPW85OZ/3dTUVK1evbraYy1atLC2a/sZ1dbemAgvQANacH1fPTLzOt3nXizDkNyeI3aXBNTMMOp16SZIv1/7bmEgKytLRUVF2rVrl3X2paCgQMXFxerWrVvI3++9997ThAkT9LOf+RZlPXjwYJ06zPqZpqnrrrtOXq9XixcvDgpYWVlZio2NVVFRkQYOHFjj87OysqrNGbNu3brjvmfv3r21d+9eRUVFqVOnTnWuNZxw2QhoYClD77C2DUl6apBdpQCON3jwYPXs2VNjx47VJ598oo8++kjjxo3TwIEDG+RyxxlnnKHXXntNGzdu1KeffqoxY8ZYZ0zqYvbs2Vq5cqUWLlyogwcPau/evdq7d68OHz6sxMRETZ8+XXfccYcWLVqkHTt2aMOGDcrLy9OiRYskSZMmTdKOHTs0bdo0bdu2TS+++KI1Uqk2gwcPVm5urkaNGqXly5dr586d+uCDDzRz5kyrU3K4I7wADezmAadrmTfH6vti7t5A3xeggfiHIbds2VIDBgzQ4MGDlZmZqZdffrlB3u/xxx9Xy5Yt1b9/f40cOVLDhg1T79696/z8NWvW6ODBg+rfv79SU1Otm7/ehx9+WA888IDmzJmjbt26adiwYXrzzTfVuXNnSVLHjh316quv6s0331R2drYWLFig3/3ud8d9T8MwtGzZMg0YMEA33nijunbtqmuvvVY7d+60+vmEO8M8mQtmYaykpETJyckqLi5WUlKS3eUAkqSJz+VrYeEQuQ3/5C/x0sw99haFJu3IkSMqLCxU586dFRcXZ3c5aCKO93tXn7/fnHkBGsGfJ+TojxWjrekszIpDnH0BgJNEeAEayWcZE+Sp/MoZkvT2w7bWAwCRivACNJIXbu6nP1cEFmPzHD1gYzUAELkIL0AjMQxDb7T9lXXpyOWt4NIRAJwEwgvQiBZP7KfyyumVDEPS8pn2FgQAEYjwAjSi1s1jNafil8EddwEbOWzAKcJcqH7fCC9AI9ucPsbquCtT0issF4DG55/y/tAhAjQaj3/ZBLfbfUqvw/IAQCNbfNN5WvzgYN0Q9S/fpaPNr0hX0/cFjcvtdqtFixbWgobx8fE1rgkEhIrX69V3332n+Ph4RUWdWvwgvACNLC7arTme6zXB7QsvpiqHTgONrH379pJkBRigoblcLnXs2PGUgzLhBbDB2NzTVbHerWh5ApeOOPuCRmYYhlJTU9WuXTuVl5fbXQ6agJiYGLlcp95jhfAC2GDmiCzN//ByTYl+o/LS0auEF9jG7Xafch8EoDHRYRewgdtl6DHPaHn9o45kMucLANQR4QWwyXmdW6lMvhEfLBcAAHVHeAFssujG8/RcxdBAQ8UR+4oBgAhCeAFsEhft1h+9v7QuHXkrjtpbEABECMILYKOf9U63hkkbppd+LwBQB4QXwEb3XZaltd5ukirXOqLfCwCcEOEFsFHLhBjdUH6PtdaRyg7YWg8ARALCC2C3qDhr0/RWcOkIAE6A8ALYbHz/DH1mdpbEkGkAqAvCC2Czu4edpcnltwcaGDINAMdFeAFsFuV2abfaMWQaAOqI8AKEgZHZaYEh02LINAAcD+EFCANTB3fVOu9Zkuj3AgAnQngBwkCn1vGaWH5XYMh0xWFb6wGAcEZ4AcKAYRg6bDST13/xyOuxtyAACGOEFyBMxEa5VaYoSZLpLaffCwDUImzDy6FDh5SRkaHp06fbXQrQKH41MFMvei6RVNnvZflMW+sBgHAVtuHlv/7rv3TeeefZXQbQaG6/pIv+UHFtlX4vh2ytBwDCVViGly+//FKff/65LrvsMrtLARqNYRjyumOtfi+mxKUjAKhBvcPLu+++q5EjRyotLU2GYWjJkiXV9nnyySfVuXNnxcXFqU+fPnrvvffq9R7Tp0/XnDlz6lsaEPGG90jVO95ekhgyDQC1qXd4KS0tVXZ2tubNm1fj4y+//LKmTp2q++67Txs2bNCFF16o4cOHq6ioyNqnT58+6t69e7Xb7t279Y9//ENdu3ZV165d61TP0aNHVVJSEnQDItVvBp2upz1Vzjh6mG0XAI4VVd8nDB8+XMOHD6/18blz5+qmm27SxIkTJUlPPPGEli9frvnz51tnU9avX1/r89etW6eXXnpJf//733Xw4EGVl5crKSlJDzzwQI37z5kzRw8++GB9DwMIS13aJWqj93SZpmQYkkyv3SUBQNgJaZ+XsrIyrV+/XkOHDg1qHzp0qD744IM6vcacOXO0a9cu7dy5U4899phuvvnmWoOLJM2YMUPFxcXWbdeuXad0DICd3C5DFa44q9+Lx8N8LwBwrJCGl++//14ej0cpKSlB7SkpKdq7d28o38oSGxurpKSkoBsQyS7s0kbllSdFXSbzvQDAsep92aguDMMIum+aZrW2upgwYUKIKgIix7j+nfTqjgs1NuqdQKfdnJvsLgsAwkZIz7y0adNGbre72lmWffv2VTsbA6BmvdNb6omKqwMNFUfsKwYAwlBIw0tMTIz69OmjFStWBLWvWLFC/fv3D+VbAY6VHB+t79VCXv9kdXTaBYAg9b5sdPDgQW3fvt26X1hYqI0bN6pVq1bq2LGjpk2bpuuvv159+/ZVbm6unnrqKRUVFWnSpEkhLRxwsmi3oXJFKVYVMj1HZeQ/zaUjAKhU7/Dy8ccf66KLLrLuT5s2TZI0fvx4Pffccxo9erR++OEHPfTQQ9qzZ4+6d++uZcuWKSMjI3RVAw53WY9ULdtyrn7m/oB+LwBwDMM0rZVUIlpeXp7y8vLk8Xj0xRdfqLi4mJFHiFib/1Ose+Y9r6Wxv/U1RMVLM/fYWxQANKCSkhIlJyfX6e93WK5tdDImT56sgoIC5efn210KcMq6piTqC3UILNIo+r0AgJ9jwgvgJDFRLnkUZU1WJy/hBQD8CC9AmIpyGdZkdaa3jMnqAKAS4QUIU5f1SNU/PL4pBlhhGgACCC9AmLp5QKae9FwZaGCyOgCQRHgBwlbXlER9baYEJqtzN8hqHgAQcQgvQJiKdrtkyND3ZrKvofUZ9hYEAGHCMeElLy9PWVlZysnJsbsUIGSi3YbijDJJkrl7A512AUAOCi/M8wInurxnqv7hOV8SnXYBwM8x4QVwookXZmpexahAA512AYDwAoSzLu0S9a1aBjrtMtMuABBegHAWE+XrtGsy0y4AWAgvQJiLcledabecTrsAmjzCCxDmLjmrnZZU7bT7/uO21gMAdiO8AGFuzHkZ+n+eyyVJpiR1ONfWegDAboQXIMx1S01SoZkqr1l55mX7CrtLAgBbOSa8MEkdnKptYqxMuQKddssZLg2gaXNMeGGSOjiZYUgVcvvueMvotAugSXNMeAGcLNrt0jLveYEGZtoF0IQRXoAIMLJnqp6pGB5oMOyrBQDsRngBIsCNF2TqC7NDYKbdjrm21gMAdiK8ABHgjHbNdVQxgU67X62xtyAAsBHhBYgAvmUCqnTaLS+l0y6AJovwAkSIKLehVd5zAg102gXQRBFegAhxYZe2etlzUaCBTrsAmijCCxAhftGng7Z50yVVLhOQedFx9wcAp3JMeGGGXThdl5Tm2q3WVZYJWGl3SQBgC8eEF2bYhdNltE6QZLBMAIAmzzHhBXC6aLdLibFRLBMAoMkjvAARpMLr1XJv30DD+4/bVwwA2ITwAkSQ3NNb64WKIZIqO+2mn3fc/QHAiQgvQAS5uk+6tpkdJNFpF0DTRXgBIkiXds21X4mBNY4qDttaDwDYgfACRJBObRIkSR5/p13DbWM1AGAPwgsQQaLdLrkMaZ23m6/B62HEEYAmh/ACRJj2SXH6h/d83x3PUdY4AtDkEF6ACHPgaIU+r1wmQJIvwABAE0J4ASLMZT1S9aXZQaa/06472tZ6AKCxEV6ACDM+t5OOKkalZqyvITXb3oIAoJE5JrywMCOais6VI45ijHJfwzcf21gNADQ+x4QXFmZEU9Esxi1Dktc/XLr8ECOOADQpjgkvQFPidhla6q2yNABrHAFoQggvQATqm9FSz7PGEYAmivACRKDLeqZqu3maJNY4AtD0EF6ACJTZprkOKD6wxpHpsbUeAGhMhBcgAmW29Y04+sFM8jW0OdPGagCgcRFegAjUPilOktTMKPM1/Gc9I44ANBmEFyACuVyGXIb0trdXZYvJGkcAmgzCCxCh2ifH6SXPxYEGw75aAKAxEV6ACHXgSIW2e30jjkxJ6jzIxmoAoPEQXoAIdWn39vpOyfKalSdddjBcGkDTQHgBItT43E6SDHn9X2Ov185yAKDREF6ACOVfoPEbs7WvoUVHG6sBgMZDeAEiVEJslAxJLYxDvobvtjJcGkCT4JjwkpeXp6ysLOXk5NhdCtBoWifE6B+e/oEGhksDaAIcE14mT56sgoIC5efn210K0GgOl3v0/zwj7C4DABqVY8IL0BQN6NpW/zFbq8Ks/Cqnn2tvQQDQCAgvQAS7qncHmXKpcqYXaef7ttYDAI2B8AJEsM5t4iUpMFzaZLg0AOcjvAARrENLX3h539vD12B6GHEEwPEIL0AEi4t2q1m0S0s85/saPGWMOALgeIQXIMKVe0x9ZaYGGligEYDDEV6ACOcypMKq4aXThfYVAwCNgPACRLhLuqWoVM3krRxwpK9W2VoPADQ0wgsQ4a7q3UGS5LEWaPTYWA0ANDzCCxDh/MOlN3i7+BpMLyOOADga4QWIcP7h0su9let6VRxhxBEARyO8ABEuLtotQ9IORhwBaCIIL4ADtG4eo6/MtEBD5sX2FQMADSzK7gIAnLrDZR79aLaVaUqGIWn7CrtLAoAGw5kXwAEGdG0rr1zy+q8XscYRAAcjvAAO4B8u/a3Z0tfQ6gwbqwGAhkV4ARzAP1y6uXHY17D3U4ZLA3Asx4SXvLw8ZWVlKScnx+5SgEbnHy69wtu3ssWU3n/cvoIAoAE5JrxMnjxZBQUFys/Pt7sUoNH5h0u/UHFJoDH9PNvqAYCG5JjwAjR1bpcRvLr09pX2FQMADYjwAjhEr44ttF+JgQUaGXEEwKEIL4BDDDu7vSTpJzPR19Cum43VAEDDIbwADpHROkGSFGeU+Rp2fcSIIwCORHgBHCKjtW/E0XveHpUtjDgC4EyEF8AhOrbyhZfXPBcGGhlxBMCBCC+AQ8RFuxUb5dKOqgs0bn/bvoIAoIEQXgAHqfCYKjJTZPpHHIkRRwCch/ACOIjLJZUrSqWK8zWkZttbEAA0AMIL4CADurSVJMWqcsTRNx/bWA0ANAzCC+AgV5zj6+/ikdvX4PXYWA0ANAzCC+Ag/hFHb3krFyg1Pcz1AsBxCC+Ag/gnqnvJc7GvwVshvf2wjRUBQOgRXgAHaRkfLUn6yltlgUaZNe8MABGK8AI4iGEYSoqL0j61UIX/651+rr1FAUCIEV4AhynzeCUZgYad/7atFgBoCIQXwGFyOrWSJHnNyq+3yUR1AJyF8AI4zPDuvv4u1gKNjDgC4DCEF8Bh/KtL/8Nzvq/BU8aIIwCOQngBHMY/18sOs8qII6OWnQEgAhFeAIdJa9FMklRYNbx0utCmagAg9AgvgMO4XYYSYtw6pDh5/adcvlplb1EAEEKEF8CBjlb4Rhh5zMrwwhpHAByE8AI4kMvwhZb/mL5VptWio43VAEBoEV4AB7qkWztJUgvjoK/hu88ZLg3AMQgvgANd1buDJOkfnv6Bxvcft6kaAAgtx4SXvLw8ZWVlKScnx+5SANv5h0s/7bks0NiBNY4AOINjwsvkyZNVUFCg/Px8u0sBbOcPL/8x2wTWlN6+wrZ6ACCUHBNeAAQ0i3HLZUgeuQPDpVnjCIBDEF4Ah2rRLFqS9JPhW6hRrc+wsRoACB3CC+BQRyrneon1HvI17PmUEUcAHIHwAjhUbmZrSdK/vH0qW0xGHAFwBMIL4FBXnJMmSfprxcWBxvTzbKoGAEKH8AI4lH/E0VdmWqBx+0qbqgGA0CG8AA7lDy8/KjEwXJoRRwAcgPACOFSrhJjKLUP7zea+zXZZttUDAKFCeAEcyjAMuSqneIlVuW9j14eMOAIQ8QgvgIOlJMVJkj6NOaeyhRFHACIf4QVwsANHKiRJfyurskAjI44ARDjCC+Bgl3RrJ0naVtE+0Lj9bZuqAYDQILwADnZN33RJ0g5PSmDEkRhxBCCyEV4AB/MPlz6qGB1SrK8xtZeNFQHAqSO8AA6WmhznX1NasfL1f9E3H9lWDwCEAuEFcLAot0uuyvHSHv/X3fTYWBEAnDrCC+Bw/snqPort52vwepnrBUBEI7wADnfoqO9y0VMHLvA1eMultx+2sSIAODWEF8DhBnRtK0na4U21uRIACA3CC+BwP+t1miRpj1oF+r0wUR2ACEZ4ARwuo3WCJMmUS/LP9rLzPfsKAoBTRHgBHM4/14skmXJXbjBRHYDIRXgBHK5ZjNua6+UDs7tvw/Qw4ghAxCK8AE2Af66Xpeb5vgZPGSOOAEQswgvQBJzToYUk6StvlQUajZr3BYBwR3gBmoBLsmpYXbrzQJuqAYBTQ3gBmoBOlSOOSpQQWFN6x9u21QMAp4LwAjQBGa0DI468/q+9lxFHACIT4QVoAvxzvUjSbvlm3FWLDJuqAYBTQ3gBmoDmsVGKifL10G1tlPoav9vKcGkAEYnwAjQVlZPrLjNzAw3vP25bOQBwsggvQBPRLilOkrQ88WeBRtY4AhCBCC9AE1F8uFyS9O53zf0nYaQvV9hWDwCcLMIL0ERc2t03x0uZogLhpeKIbfUAwMkivABNxHXnBUYXef0LNLr4JwBA5OFfLqCJ6FRluPRHOtu34fUy4ghAxCG8AE1Ecny0tb3MvMC34TnKAo0AIg7hBWhCWjSLkiR5WncNNLJAI4AIQ3gBmpCjFb6uuqt+bBlozLjQpmoA4OSEZXiJiorSOeeco3POOUcTJ060uxzAMfpltpIk/VgeI6//lMtXq2ysCADqL8ruAmrSokULbdy40e4yAMcZ0TNNq7Z9pzKPKdPfBYbh0gAiTFieeQHQMDq1CawuXeH/bxeDfwYARJZ6/6v17rvvauTIkUpLS5NhGFqyZEm1fZ588kl17txZcXFx6tOnj9577716vUdJSYn69OmjCy64QGvWrKlviQBqUXV16Y2JA30bpofh0gAiSr3DS2lpqbKzszVv3rwaH3/55Zc1depU3XfffdqwYYMuvPBCDR8+XEVFRdY+ffr0Uffu3avddu/eLUnauXOn1q9frwULFmjcuHEqKSk5ycMDUFXrhBhr+08lg3wb3gqGSwOIKPXu8zJ8+HANHz681sfnzp2rm266yepo+8QTT2j58uWaP3++5syZI0lav379cd8jLS1NktS9e3dlZWXpiy++UN++fWvc9+jRozp69Kh1n6AD1M4wDLkMyWtK2820KsOkzeM9DQDCSkgvdpeVlWn9+vUaOnRoUPvQoUP1wQcf1Ok1fvrpJyuMfPPNNyooKFBmZmat+8+ZM0fJycnWLT09/eQPAGgCUipXl27dpp0U5dtWWi8bKwKA+glpePn+++/l8XiUkpIS1J6SkqK9e/fW6TW2bt2qvn37Kjs7WyNGjND//M//qFWrVrXuP2PGDBUXF1u3Xbt2ndIxAE534EiFJGnn96W+S0aStOtDGysCgPppkKHShhE8ZadpmtXaatO/f39t2rSpzu8VGxur2NjYetUHNGVDstrp9Q27dbTCq4ool+8fgfJDvk67OTfZXR4AnFBIz7y0adNGbre72lmWffv2VTsbA8Aeo3M6SpIqvKaWe88NPPD+4zZVBAD1E9LwEhMToz59+mjFihVB7StWrFD//v1D+VYATlLV1aX/rir909LPs6EaAKi/el82OnjwoLZv327dLyws1MaNG9WqVSt17NhR06ZN0/XXX6++ffsqNzdXTz31lIqKijRp0qSQFg7g5KQkxcptSB5Tat7hbOmbyge2r7S1LgCoq3qHl48//lgXXXSRdX/atGmSpPHjx+u5557T6NGj9cMPP+ihhx7Snj171L17dy1btkwZGRmhqxrASTMMwxoY/U5RReD8a8Vhu0oCgHqpd3gZNGiQTPP4c0L85je/0W9+85uTLupk5OXlKS8vTx6Pp1HfF4hEbpchr8eU1zQlV5Rv1JHhtrssAKgTxyxqMnnyZBUUFCg/P9/uUoCwd8EZbSRJXq+p4jjfpJBK7mBjRQBQd44JLwDqblSv0yTJt7r0oR99jd9/wRpHACIC4QVogjLbNLe2X9fFgQdY4whABCC8AE1Q57ZVVpfu/KvAA3TaBRABCC9AE9Q8Nspak/Htr0olo/KfAleDTLoNACFFeAGaKJfLF1+8pik1b+9rbHW6jRUBQN0QXoAmqkOLZpKk01rESWUHfY17P6PTLoCw55jwkpeXp6ysLOXk5NhdChAR9h8ukyR9ua9UX7T1LxNg0mkXQNhzTHhhnhegfq7MPs3anrnv4uPsCQDhxTHhBUD93HhBZ2u7fcaZgU676efW8gwACA+EF6CJ6tCymbW96ssfJP/4o53v21MQANQR4QVooqLcLlUOOKpc46hybaPyUjrtAghrhBegCWubGCtJ6tw6QTprROABOu0CCGOEF6AJKz3qW4V9y+4S/SuhSnjxHLWpIgA4McIL0IQNyUqRJJmSHv7ICDzgjrGnIACoA8IL0ISNPa+jtX3AaC5Fx/vupPa0qSIAODHHhBcmqQPq74x2gdWl+5/eWvKU++7s+simigDgxBwTXpikDqi/FvEx1gKN7375fWDEUcURRhwBCFuOCS8ATo67cry0x2tKvcYFHmDEEYAwRXgBmrj0Vr7J6tJaxEkD7w48UHHEpooA4PgIL0AT91Opr5/Lju9K9fzmQ4EHTK9NFQHA8RFegCbu6r4drO3Hlm+TXNG+Oy7+eQAQnvjXCWjiJg86w9o2Jen0S3x3vB467QIIS4QXoIlrmRCjmCjfPwU9OyRLOTf6HvCU0WkXQFgivACQ12tKkj7e+aOUmh14gE67AMIQ4QWAXIZvuPThcq+e31J1XSM67QIIP44JL8ywC5y8S7u3t7bnr94hGZX/NHgJLwDCj2PCCzPsAiev6hpHfTJaBhZm9JbTaRdA2HFMeAFw8rqmJFrba7btk/reGHiQTrsAwgzhBYBaJsRY20cqvNJFvw08WHHYhooAoHaEFwCSAmscuQxDik2U/Es2MtMugDBDeAEgSTqvUytJktc09fy6ryV35Uy7njL6vQAIK4QXAJKkn/U+TZJ0tMLrWybg7KsCD9LvBUAYIbwAkCR1S02yto9WeKQLpwUerDhawzMAwB6EFwCSpC4pza1t05TUukuVRz2NXg8A1IbwAkCSFBvlVmWfXV/nXZeLyeoAhCXCCwBLWotmkqT0lvG+BiarAxCGCC8ALCWHyyVJn397wDfiqM8NgQfptAsgTBBeAFiu6t3B2n5s+TbpkgcCDzJZHYAw4ZjwwsKMwKmbcvEZ1vaRCo8UEy9rsjovnXYBhAfHhBcWZgROXZvmsda2aVZuRFW20e8FQJhwTHgBEBpRVZcJkKTe4wIP0u8FQBggvAAI0v/01pIkj9fr67RLvxcAYYbwAiDIted2lCSVeUxfp92qizRWHOHSEQDbEV4ABOnZIdnaPlJR2Uk3KiawA5eOANiM8AIgyGmVE9VJVTrt9pkQ2MHDOkcA7EV4ARDEMAxrmQCvtzK9DH4wsIPJUgEA7EV4AVBNtNv3T0O51/R12o1uVmWdI+Z7AWAvwguAaq7pe8xMu1LwfC+v3GRDVQDgQ3gBUM3tg7ta21an3dxbAztsfrWRKwKAAMILgGqqzrTr9ffavei3gR0MdyNXBAABhBcANYp2+3rtllVU9nsxDMldOWTarGC+FwC2IbwAqNHFZ7Wztq1+L9ljAjssn9nIFQGAD+EFQI1uvjDT2j7q7/dy6e8CO1QcauSKAMDHMeElLy9PWVlZysnJsbsUwBF6VJlp1z/di2ISZC0VIHHpCIAtHBNeJk+erIKCAuXn59tdCuAIsVFuGcdOVif55nzxY6kAADZwTHgBEHoxx05WJ0mDZgR2YJVpADYgvACo1ei+6da21Wn3/NsCs+2yyjQAGxBeANTqzqFnWttHyqssCxAVmAeGS0cAGhvhBUCtkuOjre65HrNKv5eB9wa2yw40ak0AQHgBcFz+yerKPVX6vVwwNbCDlwnrADQuwguA4/pZrxoWaZQCs+1KXDoC0KgILwCO6+5La+n3MvjBwDaXjgA0IsILgONq3Ty25n4vub8JbHsrpFduatS6ADRdhBcAJxQTVTnfi8fUbX/dEHig84DA9uZXG7kqAE0V4QXACf3y3I7W9puf7g48MO6NKnuZdNwF0CgILwBO6N7hZ1nbbleVtY0MQ4qqslwAK00DaASEFwAnFBftlj+zGMc+eNF9gW1WmgbQCAgvAOrE6vfiPabfy/m3Bu9Ix10ADYzwAqBOJg083doO6vciSV2GBrY3v9JIFQFoqggvAOpk6uCu1rbr2GtHY/8efJ+zLwAaEOEFQJ3FVl468pgKvnQkSZkXBbY5+wKgARFeANTZuNwMa7vapaNxS4Lvc/YFQANxTHjJy8tTVlaWcnJy7C4FcKx7h3cL3Kk27EhSVGxgm7MvABqIY8LL5MmTVVBQoPz8fLtLARyr6hwvpqnAKtN+w+YE339qUMMXBaDJcUx4AdA4+me2trYfWVoQ/GDOTdIZVUYe7T6mXwwAhADhBUC9LJ54nrV9pNxbfYfrjhl5tGBgA1cEoKkhvACoF7fLUFSVy0fVRh1JUtaVge29G+m8CyCkCC8A6u2WAZnW9hvHjjqSpGv+IhnuwP3Nr7BoI4CQIbwAqLe7hp154p0u+2Pw/X/e0zDFAGhyCC8A6s0wDEW7q146+qT6Tjk3SWddEbjvLefyEYCQILwAOCl3X3qWtf3Gp3tq3unaxZIrKnB/8ysEGACnjPAC4KTcfGFm0P0aO+5K0vA/BN+n/wuAU0R4AXDSLuzSxtquseOu5Lt81P3q4Lal0wgwAE4a4QXASVt803lB92s9+3L101LqOcFty6Y3TFEAHI/wAuCU9MtsZW3XevZFkn61RmqfHbhveqUHW3IGBkC9EV4AnJK/3twv6H619Y6qmvSulHRa4L7p5RISgHojvAA4JYZhKKbKsOnZb2w5/hOmFUhJHYLblk5jGQEAdUZ4AXDKHhiZZW1XeM3a+774TdsiJaUHt+3dKP2uQ427A0BVhBcAp+y6fp2U2SbBun/cvi9+0zZLab2C28oOSLOTpf/uFuIKATgJ4QVASLwzfVDQ/Rpn3T3WLauly+dKMoLbD+z2hZgFF4aqPAAOQngBEDJnpjS3tt/4dM/xO+/65dwkzd4vJadXf2zvZ74QM79/6IoEEPEILwBCZvkdwZ1uT9h5t6o7NvvOwhg1/LP07RZfiJndUlr75ClWCSDSEV4AhNTInqnWdoXX1Mg/vVf3J+fcJM36qfqMvBavtHxGZZBJllY/KpnmqRUMIOIYpumsb35JSYmSk5NVXFyspKQku8sBmqQzfrtMFd7APy1XZKfpf3/Z6zjPqEX+09LSOyXV8Z+pzEHSNYulOL77QKSpz99vwguAkHt+3deauWRzUNtJBxi/x7tLxbvq/7y4ltLP/yy1SPf1q4mJP/kaADQYwgvhBbBdTQHmkVHddV2/jFN/8acGSbtPMJdMfcQmS5f+TmrWynfWJjax8pbku0XFhO69ANSI8EJ4AcLCrS9+ojc/2xPUFrIAU1Wow0woxCRK/X4tuaMll1tyRVXeqtx3R1e2uSvbK/cxXJJhHPP/VW6q2mZU30fHPueYfVR5X5I1TN2oOlz92LaT3Ud12CdU71XD+56Mmuo/+RcL4UuFYV2u0HabJbwQXoCwMeJ/39Pm3SVBbQ0SYI6V/7Rv5WrT27DvAzRVl8/1dbIPEcIL4QUIKzUFmFPuAxMKBBzg5CWn+6Y4CBHCC+EFCDv957yt3cVHgtrOTkvS0tscNosugQhNgiFd/t+ceTlVeXl5ysvLk8fj0RdffEF4AcLQFX96T5/9J/gMTEKsW1sevNSmigCEiyYZXvw48wKEtyv+9L4++09xtfaUpFh9+NvBNlQEIBzU5+83M+wCaFRv3HqBHhnVvdp4h29LjqrTvUs17PE1ttQFIHJw5gWAbc7//dv6z/4jNT7mMqRZI8/W+P6dGrcoALbgshHhBYgo5zy4XPsPV9T6uGFI913WTRMvzGzEqgA0JsIL4QWIOM+v+1r3L9lcp1WMOraK119v6ae05DgZIZ28C4BdCC+EFyCiHe9yUm0MSaNz0jU6J12ntWimNs1j5XIRbIBIQXghvACOcTJB5ngSY6N096VnKj4mSgmxbsXHRCk+xm3db1a5HeN2KdptcGYHaCSEF8IL4Ej1ubQEoGGFepkPwgvhBWhSQn12BsCJndaimf5978Uhe736/P2OCtm7AoBN/n3vJSfc5/l1X+uBf2yW11H/uQbY59eDTrftvTnzAgAAbMcMuwAAwLEILwAAIKIQXgAAQEQhvAAAgIhCeAEAABGF8AIAACIK4QUAAEQUwgsAAIgohBcAABBRCC8AACCiEF4AAEBEIbwAAICIQngBAAARJcruAkLNv0h2SUmJzZUAAIC68v/d9v8dPx7HhZcDBw5IktLT022uBAAA1NeBAweUnJx83H0Msy4RJ4J4vV7t3r1biYmJMgwjpK9dUlKi9PR07dq1S0lJSSF97XDg9OOTnH+MHF/kc/oxOv34JOcfY0Mdn2maOnDggNLS0uRyHb9Xi+POvLhcLnXo0KFB3yMpKcmRv5B+Tj8+yfnHyPFFPqcfo9OPT3L+MTbE8Z3ojIsfHXYBAEBEIbwAAICIQniph9jYWM2aNUuxsbF2l9IgnH58kvOPkeOLfE4/Rqcfn+T8YwyH43Nch10AAOBsnHkBAAARhfACAAAiCuEFAABEFMILAACIKISXOnryySfVuXNnxcXFqU+fPnrvvffsLqlO5syZo5ycHCUmJqpdu3YaNWqUtm3bFrTPhAkTZBhG0K1fv35B+xw9elS33nqr2rRpo4SEBF1xxRX65ptvGvNQajR79uxqtbdv39563DRNzZ49W2lpaWrWrJkGDRqkLVu2BL1GuB6bX6dOnaodo2EYmjx5sqTI+/zeffddjRw5UmlpaTIMQ0uWLAl6PFSf2U8//aTrr79eycnJSk5O1vXXX6/9+/c38NH5HO8Yy8vLdc8996hHjx5KSEhQWlqaxo0bp927dwe9xqBBg6p9rtdee23QPnYd44k+w1D9Tobr8dX0fTQMQ3/84x+tfcL586vL34Vw/x4SXurg5Zdf1tSpU3Xfffdpw4YNuvDCCzV8+HAVFRXZXdoJrVmzRpMnT9a6deu0YsUKVVRUaOjQoSotLQ3a79JLL9WePXus27Jly4Ienzp1ql5//XW99NJLev/993Xw4EGNGDFCHo+nMQ+nRmeffXZQ7Zs2bbIe+8Mf/qC5c+dq3rx5ys/PV/v27TVkyBBrDSwpvI9NkvLz84OOb8WKFZKkX/ziF9Y+kfT5lZaWKjs7W/Pmzavx8VB9ZmPGjNHGjRv11ltv6a233tLGjRt1/fXXN/jxScc/xkOHDumTTz7R/fffr08++USvvfaavvjiC11xxRXV9r355puDPteFCxcGPW7XMZ7oM5RC8zsZrsdX9bj27NmjZ555RoZh6Oc//3nQfuH6+dXl70LYfw9NnNC5555rTpo0KajtrLPOMu+9916bKjp5+/btMyWZa9assdrGjx9vXnnllbU+Z//+/WZ0dLT50ksvWW3/+c9/TJfLZb711lsNWe4JzZo1y8zOzq7xMa/Xa7Zv3978/e9/b7UdOXLETE5ONhcsWGCaZngfW21uv/128/TTTze9Xq9pmpH9+UkyX3/9det+qD6zgoICU5K5bt06a5+1a9eakszPP/+8gY8q2LHHWJOPPvrIlGR+/fXXVtvAgQPN22+/vdbnhMsx1nR8ofidDOfjO9aVV15pXnzxxUFtkfL5mWb1vwuR8D3kzMsJlJWVaf369Ro6dGhQ+9ChQ/XBBx/YVNXJKy4uliS1atUqqH316tVq166dunbtqptvvln79u2zHlu/fr3Ky8uDfgZpaWnq3r17WPwMvvzyS6Wlpalz58669tpr9dVXX0mSCgsLtXfv3qC6Y2NjNXDgQKvucD+2Y5WVlen555/XjTfeGLTwaCR/flWF6jNbu3atkpOTdd5551n79OvXT8nJyWF3zJLve2kYhlq0aBHU/sILL6hNmzY6++yzNX369KD/6g33YzzV38lwPz6/b7/9VkuXLtVNN91U7bFI+fyO/bsQCd9Dxy3MGGrff/+9PB6PUlJSgtpTUlK0d+9em6o6OaZpatq0abrgggvUvXt3q3348OH6xS9+oYyMDBUWFur+++/XxRdfrPXr1ys2NlZ79+5VTEyMWrZsGfR64fAzOO+88/SXv/xFXbt21bfffqtHHnlE/fv315YtW6zaavrsvv76a0kK62OryZIlS7R//35NmDDBaovkz+9YofrM9u7dq3bt2lV7/Xbt2oXdMR85ckT33nuvxowZE7TI3dixY9W5c2e1b99emzdv1owZM/Tpp59alw3D+RhD8TsZzsdX1aJFi5SYmKirrroqqD1SPr+a/i5EwveQ8FJHVf8rV/J94Me2hbspU6bos88+0/vvvx/UPnr0aGu7e/fu6tu3rzIyMrR06dJqX8iqwuFnMHz4cGu7R48eys3N1emnn65FixZZHQRP5rMLh2OrydNPP63hw4crLS3Naovkz682ofjMato/3I65vLxc1157rbxer5588smgx26++WZru3v37urSpYv69u2rTz75RL1795YUvscYqt/JcD2+qp555hmNHTtWcXFxQe2R8vnV9ndBCu/vIZeNTqBNmzZyu93VUuK+ffuqpdJwduutt+qNN97QqlWr1KFDh+Pum5qaqoyMDH355ZeSpPbt26usrEw//fRT0H7h+DNISEhQjx499OWXX1qjjo732UXSsX399ddauXKlJk6ceNz9IvnzC9Vn1r59e3377bfVXv+7774Lm2MuLy/XNddco8LCQq1YsSLorEtNevfurejo6KDPNdyP0e9kficj4fjee+89bdu27YTfSSk8P7/a/i5EwveQ8HICMTEx6tOnj3Wqz2/FihXq37+/TVXVnWmamjJlil577TW988476ty58wmf88MPP2jXrl1KTU2VJPXp00fR0dFBP4M9e/Zo8+bNYfczOHr0qLZu3arU1FTrlG3VusvKyrRmzRqr7kg6tmeffVbt2rXT5Zdfftz9IvnzC9Vnlpubq+LiYn300UfWPh9++KGKi4vD4pj9weXLL7/UypUr1bp16xM+Z8uWLSovL7c+13A/xqpO5ncyEo7v6aefVp8+fZSdnX3CfcPp8zvR34WI+B6eUnffJuKll14yo6OjzaefftosKCgwp06daiYkJJg7d+60u7QT+vWvf20mJyebq1evNvfs2WPdDh06ZJqmaR44cMC88847zQ8++MAsLCw0V61aZebm5pqnnXaaWVJSYr3OpEmTzA4dOpgrV640P/nkE/Piiy82s7OzzYqKCrsOzTRN07zzzjvN1atXm1999ZW5bt06c8SIEWZiYqL12fz+9783k5OTzddee83ctGmT+ctf/tJMTU2NiGOryuPxmB07djTvueeeoPZI/PwOHDhgbtiwwdywYYMpyZw7d665YcMGa6RNqD6zSy+91OzZs6e5du1ac+3atWaPHj3MESNG2H6M5eXl5hVXXGF26NDB3LhxY9D38ujRo6Zpmub27dvNBx980MzPzzcLCwvNpUuXmmeddZbZq1evsDjG4x1fKH8nw/H4/IqLi834+Hhz/vz51Z4f7p/fif4umGb4fw8JL3WUl5dnZmRkmDExMWbv3r2DhhqHM0k13p599lnTNE3z0KFD5tChQ822bdua0dHRZseOHc3x48ebRUVFQa9z+PBhc8qUKWarVq3MZs2amSNGjKi2jx1Gjx5tpqammtHR0WZaWpp51VVXmVu2bLEe93q95qxZs8z27dubsbGx5oABA8xNmzYFvUa4HltVy5cvNyWZ27ZtC2qPxM9v1apVNf5Ojh8/3jTN0H1mP/zwgzl27FgzMTHRTExMNMeOHWv+9NNPth9jYWFhrd/LVatWmaZpmkVFReaAAQPMVq1amTExMebpp59u3nbbbeYPP/wQFsd4vOML5e9kOB6f38KFC81mzZqZ+/fvr/b8cP/8TvR3wTTD/3toVB4IAABARKDPCwAAiCiEFwAAEFEILwAAIKIQXgAAQEQhvAAAgIhCeAEAABGF8AIAACIK4QUAAEQUwgsAAIgohBcAABBRCC8AACCiEF4AAEBE+f+JZEzxuZh6LAAAAABJRU5ErkJggg==",
      "text/plain": [
       "Figure(PyObject <Figure size 640x480 with 1 Axes>)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "PyObject <matplotlib.legend.Legend object at 0x0000000064143EE0>"
      ]
     },
     "execution_count": 127,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "plot(losses, \"-o\", ms=1, label=\"normalized loss\")\n",
    "plot(errs, \"-o\", ms=1, label=\"normalized err\")\n",
    "yscale(\"log\")\n",
    "legend()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "100-element Array{Any,1}:\n",
       "   3.6529179730351864e279\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       "   ⋮\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN\n",
       " NaN"
      ]
     },
     "execution_count": 66,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "8×8 Array{Bool,2}:\n",
       " 1  1  1  1  1  1  1  1\n",
       " 1  1  1  0  1  1  1  1\n",
       " 1  0  1  1  1  1  1  1\n",
       " 1  1  1  1  1  1  1  1\n",
       " 1  1  1  0  1  1  1  1\n",
       " 1  1  1  1  1  0  1  1\n",
       " 1  1  1  1  1  1  1  1\n",
       " 1  1  1  1  1  1  1  1"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mask"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1-element Array{Array{Float64,2},1}:\n",
       " [-8078.541977917878 1095.3318247755824 … 10061.667922392518 4805.541908299654; -1458.9476865415784 8192.371673592666 … -5392.612976088403 -8080.408973099034; … ; -6984.488832229687 -7972.2685265714545 … 4381.108486960437 6816.586950212095; 5851.07235370321 1736.0214276731454 … -8551.897599453881 -1274.9795665158942]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Ws"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.5.3",
   "language": "julia",
   "name": "julia-1.5"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.5.3"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}