D_manipulation_detection.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Supervised learning for adversarial manipulation detection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Paths\n",
    "dataset_root = 'output/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Start importing required libraries...\n",
      "Done importing, using pytorch version 1.12.1+cu102...\n"
     ]
    }
   ],
   "source": [
    "# Importing libraries\n",
    "print('Start importing required libraries...')\n",
    "import os, sys, time, allel, yaml, math, gzip, torch\n",
    "sys.path.append('../LAI-Net/')\n",
    "from tqdm.auto import tqdm\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from collections import Counter\n",
    "from scipy.interpolate import interp1d\n",
    "from matplotlib import cm\n",
    "from matplotlib.patches import Patch\n",
    "from matplotlib.lines import Line2D\n",
    "import pickle as pkl\n",
    "import numpy as np\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "from matplotlib.colors import ListedColormap, LinearSegmentedColormap\n",
    "import sklearn\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "import xgboost as xgb\n",
    "from lightgbm import LGBMClassifier\n",
    "\n",
    "from sklearn.metrics import balanced_accuracy_score\n",
    "print('Done importing, using pytorch version {}...'.format(torch.__version__))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def load_dataset(path, verbose=True):\n",
    "    with open(path, 'rb') as handle:\n",
    "        dict_dataset = pkl.load(handle)\n",
    "\n",
    "\n",
    "    real_seqs = np.array(dict_dataset['real_seqs'])\n",
    "    if len(real_seqs.shape) == 3:\n",
    "        real_seqs = real_seqs.reshape(real_seqs.shape[0]*real_seqs.shape[1],real_seqs.shape[2])\n",
    "    fake_seqs = np.array(dict_dataset['fake_seqs'])\n",
    "    if len(fake_seqs.shape) == 3:\n",
    "        fake_seqs = fake_seqs.reshape(fake_seqs.shape[0]*fake_seqs.shape[1],fake_seqs.shape[2])\n",
    "\n",
    "\n",
    "    if verbose:\n",
    "        print('Real and fake shapes are ', real_seqs.shape, fake_seqs.shape)\n",
    "        \n",
    "    train_x = np.concatenate([real_seqs[0::2,:], fake_seqs[0::2,:]])\n",
    "    train_y = np.concatenate([np.zeros(real_seqs[0::2,:].shape[0]), np.ones(fake_seqs[0::2,:].shape[0])])\n",
    "\n",
    "    test_x = np.concatenate([real_seqs[1::2,:], fake_seqs[1::2,:]])\n",
    "    test_y = np.concatenate([np.zeros(real_seqs[1::2,:].shape[0]), np.ones(fake_seqs[1::2,:].shape[0])])\n",
    "    \n",
    "    if verbose:\n",
    "        print('Training (x,y) and testing (x,y) shapes are ', train_x.shape, train_y.shape, test_x.shape, test_y.shape)\n",
    "    \n",
    "    return train_x, train_y, test_x, test_y\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing dataset  dataset_manipulated.pkl\n",
      "Real and fake shapes are  (2880, 516800) (50, 516800)\n",
      "Training (x,y) and testing (x,y) shapes are  (1465, 516800) (1465,) (1465, 516800) (1465,)\n",
      "Accuracy is  1.0 LogReg\n",
      "Accuracy is  0.5 KNN\n",
      "Accuracy is  1.0 MLP\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dmasmont/anaconda3/envs/pytorch/lib/python3.9/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[17:12:06] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Accuracy is  0.8400000000000001 XGB\n",
      "Processing dataset  dataset_Neural-ADMIXTURE.pkl\n",
      "Real and fake shapes are  (2880, 516800) (2880, 516800)\n",
      "Training (x,y) and testing (x,y) shapes are  (2880, 516800) (2880,) (2880, 516800) (2880,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dmasmont/anaconda3/envs/pytorch/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:814: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is  0.8868055555555556 LogReg\n",
      "Accuracy is  0.5118055555555555 KNN\n",
      "Accuracy is  0.8659722222222221 MLP\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dmasmont/anaconda3/envs/pytorch/lib/python3.9/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[17:40:07] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Accuracy is  0.89375 XGB\n",
      "Processing dataset  dataset_LAI-Net.pkl\n",
      "Real and fake shapes are  (2880, 516800) (2880, 516800)\n",
      "Training (x,y) and testing (x,y) shapes are  (2880, 516800) (2880,) (2880, 516800) (2880,)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dmasmont/anaconda3/envs/pytorch/lib/python3.9/site-packages/sklearn/linear_model/_logistic.py:814: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html\n",
      "Please also refer to the documentation for alternative solver options:\n",
      "    https://scikit-learn.org/stable/modules/linear_model.html#logistic-regression\n",
      "  n_iter_i = _check_optimize_result(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy is  0.8659722222222223 LogReg\n",
      "Accuracy is  0.5048611111111111 KNN\n",
      "Accuracy is  0.8579861111111111 MLP\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dmasmont/anaconda3/envs/pytorch/lib/python3.9/site-packages/xgboost/sklearn.py:1224: UserWarning: The use of label encoder in XGBClassifier is deprecated and will be removed in a future release. To remove this warning, do the following: 1) Pass option use_label_encoder=False when constructing XGBClassifier object; and 2) Encode your labels (y) as integers starting with 0, i.e. 0, 1, 2, ..., [num_class - 1].\n",
      "  warnings.warn(label_encoder_deprecation_msg, UserWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[17:49:18] WARNING: ../src/learner.cc:1115: Starting in XGBoost 1.3.0, the default evaluation metric used with the objective 'binary:logistic' was changed from 'error' to 'logloss'. Explicitly set eval_metric if you'd like to restore the old behavior.\n",
      "Accuracy is  0.8659722222222223 XGB\n"
     ]
    }
   ],
   "source": [
    "\n",
    "dataset_list = ['dataset_manipulated.pkl', 'dataset_Neural-ADMIXTURE.pkl', 'dataset_LAI-Net.pkl']\n",
    "\n",
    "for dataset_name in dataset_list:\n",
    "    print('Processing dataset ', dataset_name)\n",
    "    \n",
    "    path = dataset_root+dataset_name\n",
    "\n",
    "    train_x, train_y, test_x, test_y = load_dataset(path)\n",
    "    \n",
    "    method_name = ['LogReg', 'KNN', 'MLP', 'XGB']\n",
    "    method_list = [LogisticRegression(), KNeighborsClassifier(), MLPClassifier(), xgb.XGBClassifier()]\n",
    "\n",
    "\n",
    "    for method, m_name in zip(method_list, method_name):\n",
    "        method.fit(train_x, train_y)\n",
    "        y_pred = method.predict(test_x)\n",
    "        acc = balanced_accuracy_score(test_y, y_pred)\n",
    "        print('Accuracy is ', acc, m_name)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}