Chapter 5.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Support Vector Machine (SVM)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('scaler', StandardScaler()),\n",
       "                ('linear_svc', LinearSVC(C=1, loss='hinge'))])"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using dataset iris\n",
    "\n",
    "import numpy as np\n",
    "from sklearn import datasets\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.svm import LinearSVC\n",
    "\n",
    "iris = datasets.load_iris()\n",
    "X = iris[\"data\"][:,(2,3)]\n",
    "y = (iris[\"target\"] == 2).astype(np.float64)\n",
    "\n",
    "svm_clf = Pipeline([\n",
    "    (\"scaler\", StandardScaler()),\n",
    "    (\"linear_svc\", LinearSVC(C = 1, loss = \"hinge\"))  # or use SVC with kernel = \"linear\"\n",
    "    # or use SGDClassifier(loss=\"hinge\", alpha = 1/(m*C))\n",
    "])\n",
    "\n",
    "svm_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1.])"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# making prediction\n",
    "\n",
    "svm_clf.predict([[5.7, 1.7]]) # unlike logistic regression, svm do not output probabilites for each class"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Nonlinear SVM Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/anaconda3/lib/python3.8/site-packages/sklearn/svm/_base.py:976: ConvergenceWarning: Liblinear failed to converge, increase the number of iterations.\n",
      "  warnings.warn(\"Liblinear failed to converge, increase \"\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('poly_features', PolynomialFeatures(degree=3)),\n",
       "                ('scaler', StandardScaler()),\n",
       "                ('svm_clf', LinearSVC(C=10, loss='hinge'))])"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.datasets import make_moons\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "X, y = make_moons(n_samples = 100, noise = 0.15)  # generate dataset\n",
    "polynomial_svm_clf = Pipeline([\n",
    "    (\"poly_features\", PolynomialFeatures(degree = 3)),\n",
    "    (\"scaler\", StandardScaler()),\n",
    "    (\"svm_clf\", LinearSVC(C = 10, loss = \"hinge\"))\n",
    "])\n",
    "\n",
    "polynomial_svm_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Polynomial Kernel\n",
    "\n",
    "Kernel trick makes it possibel to get the same result as if having many polynomial features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('scaler', StandardScaler()),\n",
       "                ('svm_clf', SVC(C=5, coef0=1, kernel='poly'))])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "poly_kernel_svm_clf = Pipeline([\n",
    "    (\"scaler\", StandardScaler()),\n",
    "    (\"svm_clf\", SVC(kernel = \"poly\", degree = 3, coef0 = 1, C = 5)) # third degree polynomial\n",
    "]) # coef0 controls how much the model is influenced by high-degree polynimial vs. low-degree polynomials\n",
    "\n",
    "poly_kernel_svm_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gaussian RBF Kernel\n",
    "\n",
    "when training set is too large"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(steps=[('scaler', StandardScaler()),\n",
       "                ('svm_clf', SVC(C=0.001, gamma=5))])"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rbf_kernel_svm_clf = Pipeline([\n",
    "    (\"scaler\", StandardScaler()),\n",
    "    (\"svm_clf\", SVC(kernel = \"rbf\", gamma = 5, C = 0.001)) \n",
    "    # increasing gamma = bell-shaped curve narrower = instance's range of influence is smaller\n",
    "])\n",
    "\n",
    "rbf_kernel_svm_clf.fit(X, y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SVM Regression\n",
    "\n",
    "Tries to fit as many instances as possible on the street while limiting margin violations\n",
    "- width of the street is controlled by a hyperparameter, $\\epsilon$\n",
    "- training more instances within the margin does not affect the model's prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "LinearSVR(epsilon=1.5)"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import LinearSVR\n",
    "\n",
    "svm_reg = LinearSVR(epsilon = 1.5) # width is 1.5\n",
    "svm_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SVR(C=100, degree=2, kernel='poly')"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# using kernel trick\n",
    "\n",
    "from sklearn.svm import SVR\n",
    "\n",
    "svm_poly_reg = SVR(kernel = 'poly', degree = 2, C = 100, epsilon = 0.1)\n",
    "svm_poly_reg.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}