attack_notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Besitzer\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "C:\\Users\\Besitzer\\AppData\\Local\\Packages\\PythonSoftwareFoundation.Python.3.12_qbz5n2kfra8p0\\LocalCache\\local-packages\\Python312\\site-packages\\huggingface_hub\\file_download.py:1132: FutureWarning: `resume_download` is deprecated and will be removed in version 1.0.0. Downloads always resume when possible. If you want to force a new download, use `force_download=True`.\n",
      "  warnings.warn(\n",
      "You are using the default legacy behaviour of the <class 'transformers.models.t5.tokenization_t5.T5Tokenizer'>. This is expected, and simply means that the `legacy` (previous) behavior will be used so nothing changes for you. If you want to use the new behaviour, set `legacy=False`. This should only be set if you understand what it means, and thoroughly read the reason why this was added as explained in https://github.com/huggingface/transformers/pull/24565\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "# ! pip install -r requirements.txt\n",
    "# !pip install --upgrade openai\n",
    "\n",
    "import os , re, openai\n",
    "import sys\n",
    "import pandas as pd\n",
    "import nltk\n",
    "from nltk.corpus import stopwords, wordnet\n",
    "from nltk.tokenize import word_tokenize, sent_tokenize\n",
    "\n",
    "from utils import rephrase_essay, compute_similarity, mask_sentences, assess_quality,\\\n",
    "    generate_replacements_for_masked_sentences, substitute_word, read_essay_from_file,\\\n",
    "    evaluate_detection_evasion, evaluate_zerogpt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to\n",
      "[nltk_data]     C:\\Users\\Besitzer\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "[nltk_data] Downloading package stopwords to\n",
      "[nltk_data]     C:\\Users\\Besitzer\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package stopwords is already up-to-date!\n",
      "[nltk_data] Downloading package wordnet to\n",
      "[nltk_data]     C:\\Users\\Besitzer\\AppData\\Roaming\\nltk_data...\n",
      "[nltk_data]   Package wordnet is already up-to-date!\n"
     ]
    }
   ],
   "source": [
    "# Download required NLTK data\n",
    "nltk.download('punkt')\n",
    "nltk.download('stopwords')\n",
    "nltk.download('wordnet')\n",
    "\n",
    "# Initialize models with a valid API key\n",
    "openai.api_key = \"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\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000\n",
      "581\n"
     ]
    }
   ],
   "source": [
    "# This code cell is for text manipulation. \n",
    "\n",
    "student_essay_path = \"human\"\n",
    "gpt_essay_path = \"gpt_essays\"\n",
    "word_sub_essay_path = \"word_sub_essays\"\n",
    "sentence_essay_path = \"sentence_sub_essays\"\n",
    "gpt_rep_essay_path = \"gpt_rep_essays\"\n",
    "\n",
    "# Ensure the manipulated_essay_path directory exists\n",
    "if not os.path.exists(word_sub_essay_path):\n",
    "    os.makedirs(word_sub_essay_path)\n",
    "\n",
    "if not os.path.exists(sentence_essay_path):\n",
    "    os.makedirs(sentence_essay_path)\n",
    "\n",
    "if not os.path.exists(gpt_rep_essay_path):\n",
    "    os.makedirs(gpt_rep_essay_path)\n",
    "\n",
    "# Listing files from both directories\n",
    "student_essay_files = sorted([os.path.join(student_essay_path, f) for f in os.listdir(student_essay_path) if os.path.isfile(os.path.join(student_essay_path, f))])\n",
    "gpt_essay_files = sorted([os.path.join(gpt_essay_path, f) for f in os.listdir(gpt_essay_path) if os.path.isfile(os.path.join(gpt_essay_path, f))])\n",
    "\n",
    "print(len(student_essay_files))\n",
    "print(len(gpt_essay_files))\n",
    "def manipulate_gpt_essay(student_essay_files, gpt_essay_files, num_essays=6):\n",
    "    max_essays = num_essays\n",
    "    student_essay_files = student_essay_files[:max_essays]\n",
    "    gpt_essay_files = gpt_essay_files[:max_essays]\n",
    "    # Process each pair of essays to apply the desired manipulations\n",
    "    for student_file, gpt_file in zip(student_essay_files, gpt_essay_files):\n",
    "        # Read essays from files\n",
    "        # print(student_file)\n",
    "        # print(gpt_file)\n",
    "        student_essay = read_essay_from_file(student_file)\n",
    "        gpt_essay = read_essay_from_file(gpt_file)\n",
    "\n",
    "        # Rephrase the essay \n",
    "        rephrased_essay = rephrase_essay(student_essay, gpt_essay)\n",
    "        \n",
    "        # Step 3: Apply sentence-level replacement to the rephrased essay\n",
    "        masked_essay, sentences_to_replace = mask_sentences(rephrased_essay, int(len(sent_tokenize(rephrased_essay)) * 0.45))\n",
    "        sentence_substituted_essay = generate_replacements_for_masked_sentences(masked_essay, sentences_to_replace)\n",
    "        \n",
    "        # Step 4: Apply word-level replacement\n",
    "        word_substituted_essay = substitute_word(sentence_substituted_essay, 152)\n",
    "\n",
    "        # Saving the sentence substituted essay to the specified directory\n",
    "        output_file_path = os.path.join(sentence_essay_path, os.path.basename(gpt_file))\n",
    "        with open(output_file_path, 'w', encoding='utf-8') as output_file:\n",
    "            output_file.write(sentence_substituted_essay)\n",
    "\n",
    "        # Saving the word substituted essay to the specified directory\n",
    "        output_file_path = os.path.join(word_sub_essay_path, os.path.basename(gpt_file))\n",
    "        with open(output_file_path, 'w', encoding='utf-8') as output_file:\n",
    "            output_file.write(word_substituted_essay)\n",
    "\n",
    "        # Saving the gpt rephrased essay to the specified directory\n",
    "        output_file_path = os.path.join(gpt_rep_essay_path, os.path.basename(gpt_file))\n",
    "        with open(output_file_path, 'w', encoding='utf-8') as output_file:\n",
    "            output_file.write(rephrased_essay)\n",
    "\n",
    "# manipulate_gpt_essay(student_essay_files, gpt_essay_files, 25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Student Essays: ['1.txt', '10.txt', '100.txt', '101.txt', '102.txt', '103.txt', '104.txt', '105.txt', '106.txt', '107.txt', '108.txt', '109.txt', '11.txt', '110.txt', '111.txt', '112.txt', '113.txt', '114.txt', '115.txt', '116.txt', '117.txt']\n",
      "GPT Essays: ['1.txt', '10.txt', '100.txt', '101.txt', '102.txt', '103.txt', '104.txt', '105.txt', '106.txt', '107.txt', '108.txt', '109.txt', '11.txt', '110.txt', '111.txt', '112.txt', '113.txt', '114.txt', '115.txt', '116.txt', '117.txt']\n",
      "Sentence Manipulated Essays: ['1.txt', '10.txt', '100.txt', '101.txt', '102.txt', '103.txt', '104.txt', '105.txt', '106.txt', '107.txt', '108.txt', '109.txt', '11.txt', '110.txt', '111.txt', '112.txt', '113.txt', '114.txt', '115.txt', '116.txt', '117.txt']\n",
      "GPT Rephrased Essays: ['1.txt', '10.txt', '100.txt', '101.txt', '102.txt', '103.txt', '104.txt', '105.txt', '106.txt', '107.txt', '108.txt', '109.txt', '11.txt', '110.txt', '111.txt', '112.txt', '113.txt', '114.txt', '115.txt', '116.txt', '117.txt']\n",
      "Word Manipulated Essays: ['1.txt', '10.txt', '100.txt', '101.txt', '102.txt', '103.txt', '104.txt', '105.txt', '106.txt', '107.txt', '108.txt', '109.txt', '11.txt', '110.txt', '111.txt', '112.txt', '113.txt', '114.txt', '115.txt', '116.txt', '117.txt']\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Define the directories for different types of essays\n",
    "student_essay_path = \"human\"\n",
    "gpt_essay_path = \"gpt_essays\"\n",
    "word_sub_essay_path = \"word_sub_essays\"\n",
    "sentence_essay_path = \"sentence_sub_essays\"\n",
    "gpt_rep_essay_path = \"gpt_rep_essays\"\n",
    "\n",
    "# Get the list of filenames from the word substitution essays directory\n",
    "word_subs_essay_filenames = sorted([f for f in os.listdir(word_sub_essay_path)])\n",
    "\n",
    "# Filter the student and GPT essays to include only those present in the word-substituted essays\n",
    "student_essay_files = sorted([os.path.join(student_essay_path, f) for f in os.listdir(student_essay_path) if f in word_subs_essay_filenames])\n",
    "gpt_essay_files = sorted([os.path.join(gpt_essay_path, f) for f in os.listdir(gpt_essay_path) if f in word_subs_essay_filenames])\n",
    "\n",
    "# Apply similar filtering for the sentence and GPT rephrased essays directories\n",
    "sentence_essay_files = sorted([os.path.join(sentence_essay_path, f) for f in os.listdir(sentence_essay_path) if f in word_subs_essay_filenames])\n",
    "gpt_rep_essay_files = sorted([os.path.join(gpt_rep_essay_path, f) for f in os.listdir(gpt_rep_essay_path) if f in word_subs_essay_filenames])\n",
    "\n",
    "# Calculate the maximum number of essays to process based on the smallest category of available files\n",
    "max_essays = min(21, len(student_essay_files), len(gpt_essay_files), len(sentence_essay_files), len(gpt_rep_essay_files))\n",
    "\n",
    "# Limit the number of essays processed to max_essays\n",
    "student_essay_files = student_essay_files[:max_essays]\n",
    "gpt_essay_files = gpt_essay_files[:max_essays]\n",
    "sentence_essay_files = sentence_essay_files[:max_essays]\n",
    "gpt_rep_essay_files = gpt_rep_essay_files[:max_essays]\n",
    "word_sub_essay_files = word_subs_essay_filenames[:max_essays]\n",
    "\n",
    "# Verify the corresponding filenames to ensure consistency\n",
    "print(\"Student Essays:\", [os.path.basename(f) for f in student_essay_files])\n",
    "print(\"GPT Essays:\", [os.path.basename(f) for f in gpt_essay_files])\n",
    "print(\"Sentence Manipulated Essays:\", [os.path.basename(f) for f in sentence_essay_files])\n",
    "print(\"GPT Rephrased Essays:\", [os.path.basename(f) for f in gpt_rep_essay_files])\n",
    "print(\"Word Manipulated Essays:\", [os.path.basename(f) for f in word_sub_essay_files])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "# Define the directories for different types of essays\n",
    "student_essay_path = \"human\"\n",
    "gpt_essay_path = \"gpt_essays\"\n",
    "word_sub_essay_path = \"word_sub_essays\"\n",
    "sentence_essay_path = \"sentence_sub_essays\"\n",
    "gpt_rep_essay_path = \"gpt_rep_essays\"\n",
    "\n",
    "# Get the list of filenames from the word substitution essays directory\n",
    "word_subs_essay_filenames = sorted(os.listdir(word_sub_essay_path))\n",
    "\n",
    "# Filter essay files from each directory to include only those present in the word-substituted essays\n",
    "directories = [student_essay_path, gpt_essay_path, sentence_essay_path, gpt_rep_essay_path]\n",
    "essay_files = {dir_path: sorted([os.path.join(dir_path, f) for f in os.listdir(dir_path) if f in word_subs_essay_filenames]) for dir_path in directories}\n",
    "\n",
    "# Calculate the maximum number of essays to process based on the smallest category of available files\n",
    "max_essays = min(len(files) for files in essay_files.values())\n",
    "max_essays = min(21, max_essays)  # Ensure not to exceed 22 essays\n",
    "\n",
    "# Limit the number of essays processed to max_essays and prepare for processing\n",
    "essay_files = {dir_path: files[:max_essays] for dir_path, files in essay_files.items()}\n",
    "word_sub_essay_files = [os.path.join(word_sub_essay_path, f) for f in word_subs_essay_filenames][:max_essays]\n",
    "\n",
    "# Data container for the results\n",
    "similarity_quality_results = []\n",
    "\n",
    "# Process each pair of essays\n",
    "for i in range(max_essays):\n",
    "    student_essay = read_essay_from_file(essay_files[student_essay_path][i])\n",
    "    gpt_essay = read_essay_from_file(essay_files[gpt_essay_path][i])\n",
    "    sentence_substituted_essay = read_essay_from_file(essay_files[sentence_essay_path][i])\n",
    "    gpt_rep_essay = read_essay_from_file(essay_files[gpt_rep_essay_path][i])\n",
    "    word_substituted_essay = read_essay_from_file(word_sub_essay_files[i])\n",
    "\n",
    "    # Assuming these functions are defined elsewhere in the code: \n",
    "    similarity_before_word_substitution = compute_similarity(student_essay,  gpt_rep_essay)\n",
    "    similarity_after_word_substitution = compute_similarity(student_essay, word_substituted_essay)\n",
    "    quality_score_before = assess_quality( gpt_rep_essay)\n",
    "    quality_score_after = assess_quality(word_substituted_essay)\n",
    "    human_detection_metrics = evaluate_detection_evasion(student_essay)\n",
    "    gpt_detection_metrics = evaluate_detection_evasion(gpt_essay)\n",
    "    rep_detection_metrics = evaluate_detection_evasion( gpt_rep_essay)\n",
    "    ss_detection_metrics = evaluate_detection_evasion(sentence_substituted_essay)\n",
    "    ws_detection_metrics = evaluate_detection_evasion(word_substituted_essay)\n",
    "\n",
    "    # Add results for this essay pair to the list\n",
    "    similarity_quality_results.append([\n",
    "        os.path.basename(essay_files[student_essay_path][i]),\n",
    "        similarity_before_word_substitution,\n",
    "        similarity_after_word_substitution,\n",
    "        quality_score_before / 10,\n",
    "        quality_score_after / 10,\n",
    "        human_detection_metrics['ACC'],\n",
    "        gpt_detection_metrics['ACC'], \n",
    "        rep_detection_metrics['ACC'], \n",
    "        ss_detection_metrics['ACC'],\n",
    "        ws_detection_metrics['ACC'], \n",
    "    ])\n",
    "    print(similarity_quality_results)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Essay</th>\n",
       "      <th>Simhash before</th>\n",
       "      <th>Simhash after</th>\n",
       "      <th>Quality before</th>\n",
       "      <th>Quality after</th>\n",
       "      <th>Human ACC</th>\n",
       "      <th>GPT ACC</th>\n",
       "      <th>Rephrased ACC</th>\n",
       "      <th>Sentence Sub ACC</th>\n",
       "      <th>Word Sub ACC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.txt</td>\n",
       "      <td>23</td>\n",
       "      <td>26</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.174063</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.390244</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.txt</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.002701</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.301927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100.txt</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.036122</td>\n",
       "      <td>0.997396</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.366124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.txt</td>\n",
       "      <td>27</td>\n",
       "      <td>22</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.002753</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.915631</td>\n",
       "      <td>0.444828</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>102.txt</td>\n",
       "      <td>19</td>\n",
       "      <td>22</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.179750</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.641502</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>103.txt</td>\n",
       "      <td>16</td>\n",
       "      <td>20</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.022900</td>\n",
       "      <td>0.444877</td>\n",
       "      <td>0.434058</td>\n",
       "      <td>0.982252</td>\n",
       "      <td>0.357751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>104.txt</td>\n",
       "      <td>24</td>\n",
       "      <td>23</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.030476</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.392523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>105.txt</td>\n",
       "      <td>13</td>\n",
       "      <td>18</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.053595</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.261188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>106.txt</td>\n",
       "      <td>22</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.179714</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.633054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>107.txt</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.053622</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.352039</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>108.txt</td>\n",
       "      <td>17</td>\n",
       "      <td>11</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.179750</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.738342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>109.txt</td>\n",
       "      <td>25</td>\n",
       "      <td>23</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.3</td>\n",
       "      <td>0.012464</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.479744</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>11.txt</td>\n",
       "      <td>23</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.010895</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.390334</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>110.txt</td>\n",
       "      <td>17</td>\n",
       "      <td>21</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.174074</td>\n",
       "      <td>0.997396</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.301927</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>111.txt</td>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.022346</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.445394</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>112.txt</td>\n",
       "      <td>20</td>\n",
       "      <td>23</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.010830</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997396</td>\n",
       "      <td>0.332916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>113.txt</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.030476</td>\n",
       "      <td>0.997396</td>\n",
       "      <td>0.997396</td>\n",
       "      <td>0.997396</td>\n",
       "      <td>0.722698</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>114.txt</td>\n",
       "      <td>15</td>\n",
       "      <td>14</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.179620</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.339869</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>115.txt</td>\n",
       "      <td>20</td>\n",
       "      <td>19</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.036122</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.459647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>116.txt</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "      <td>0.030474</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997396</td>\n",
       "      <td>0.997756</td>\n",
       "      <td>0.890625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>117.txt</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.6</td>\n",
       "      <td>0.010889</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.997778</td>\n",
       "      <td>0.470828</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Essay  Simhash before  Simhash after  Quality before  Quality after  \\\n",
       "0     1.txt              23             26             0.8            0.4   \n",
       "1    10.txt              19             20             0.8            0.4   \n",
       "2   100.txt              13             14             0.8            0.6   \n",
       "3   101.txt              27             22             0.9            0.6   \n",
       "4   102.txt              19             22             0.9            0.6   \n",
       "5   103.txt              16             20             0.9            0.4   \n",
       "6   104.txt              24             23             0.9            0.4   \n",
       "7   105.txt              13             18             0.8            0.4   \n",
       "8   106.txt              22             20             0.8            0.4   \n",
       "9   107.txt              19             20             0.8            0.3   \n",
       "10  108.txt              17             11             0.9            0.6   \n",
       "11  109.txt              25             23             0.9            0.3   \n",
       "12   11.txt              23             20             0.8            0.6   \n",
       "13  110.txt              17             21             0.8            0.4   \n",
       "14  111.txt              20             23             0.8            0.4   \n",
       "15  112.txt              20             23             0.8            0.4   \n",
       "16  113.txt              17             15             0.8            0.6   \n",
       "17  114.txt              15             14             0.8            0.4   \n",
       "18  115.txt              20             19             0.9            0.4   \n",
       "19  116.txt              12             12             0.8            0.4   \n",
       "20  117.txt              11             10             0.9            0.6   \n",
       "\n",
       "    Human ACC   GPT ACC  Rephrased ACC  Sentence Sub ACC  Word Sub ACC  \n",
       "0    0.174063  0.997778       0.997778          0.997756      0.390244  \n",
       "1    0.002701  0.997756       0.997756          0.997778      0.301927  \n",
       "2    0.036122  0.997396       0.997778          0.997778      0.366124  \n",
       "3    0.002753  0.997756       0.997756          0.915631      0.444828  \n",
       "4    0.179750  0.997756       0.997756          0.997756      0.641502  \n",
       "5    0.022900  0.444877       0.434058          0.982252      0.357751  \n",
       "6    0.030476  0.997778       0.997778          0.997778      0.392523  \n",
       "7    0.053595  0.997756       0.997778          0.997778      0.261188  \n",
       "8    0.179714  0.997778       0.997778          0.997778      0.633054  \n",
       "9    0.053622  0.997778       0.997778          0.997778      0.352039  \n",
       "10   0.179750  0.997778       0.997778          0.997778      0.738342  \n",
       "11   0.012464  0.997778       0.997778          0.997778      0.479744  \n",
       "12   0.010895  0.997778       0.997778          0.997756      0.390334  \n",
       "13   0.174074  0.997396       0.997778          0.997756      0.301927  \n",
       "14   0.022346  0.997778       0.997778          0.997778      0.445394  \n",
       "15   0.010830  0.997778       0.997778          0.997396      0.332916  \n",
       "16   0.030476  0.997396       0.997396          0.997396      0.722698  \n",
       "17   0.179620  0.997778       0.997778          0.997778      0.339869  \n",
       "18   0.036122  0.997778       0.997778          0.997778      0.459647  \n",
       "19   0.030474  0.997778       0.997396          0.997756      0.890625  \n",
       "20   0.010889  0.997778       0.997778          0.997778      0.470828  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define DataFrame columns and convert results to a DataFrame, then save to CSV\n",
    "columns = [\n",
    "    'Essay', 'Simhash before', 'Simhash after', \n",
    "'Quality before', 'Quality after', 'Human ACC', 'GPT ACC', 'Rephrased ACC', 'Sentence Sub ACC', 'Word Sub ACC',\n",
    "]\n",
    "zero_similarity_df = pd.DataFrame(similarity_quality_results, columns=columns)\n",
    "zero_similarity_df.to_csv(\"gptzero_manipulation_results.csv\")\n",
    "zero_similarity_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quality Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Score: 6\n",
      "Score: 8\n",
      "Score: 8\n",
      "Score: 8\n",
      "Score: 4\n",
      "[['1.txt', 18, 23, 5, 6, 12, 0.6, 0.8, 0.8, 0.8, 0.4]]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Define the directories for different types of essays\n",
    "student_essay_path = \"human\"\n",
    "gpt_essay_path = \"gpt_essays\"\n",
    "word_sub_essay_path = \"word_sub_essays\"\n",
    "sentence_essay_path = \"sentence_sub_essays\"\n",
    "gpt_rep_essay_path = \"gpt_rep_essays\"\n",
    "\n",
    "# Get the list of filenames from the word substitution essays directory\n",
    "word_subs_essay_filenames = sorted(os.listdir(word_sub_essay_path))\n",
    "\n",
    "# Filter essay files from each directory to include only those present in the word-substituted essays\n",
    "directories = [student_essay_path, gpt_essay_path, sentence_essay_path, gpt_rep_essay_path]\n",
    "essay_files = {dir_path: sorted([os.path.join(dir_path, f) for f in os.listdir(dir_path) if f in word_subs_essay_filenames]) for dir_path in directories}\n",
    "\n",
    "# Calculate the maximum number of essays to process based on the smallest category of available files\n",
    "max_essays = min(len(files) for files in essay_files.values())\n",
    "max_essays = min(21, max_essays)  # Ensure not to exceed 22 essays\n",
    "\n",
    "# Limit the number of essays processed to max_essays and prepare for processing\n",
    "essay_files = {dir_path: files[:max_essays] for dir_path, files in essay_files.items()}\n",
    "word_sub_essay_files = [os.path.join(word_sub_essay_path, f) for f in word_subs_essay_filenames][:max_essays]\n",
    "\n",
    "# Data container for the results\n",
    "similarity_quality_results = []\n",
    "\n",
    "# Process each pair of essays\n",
    "for i in range(max_essays):\n",
    "    student_essay = read_essay_from_file(essay_files[student_essay_path][i])\n",
    "    gpt_essay = read_essay_from_file(essay_files[gpt_essay_path][i])\n",
    "    sentence_substituted_essay = read_essay_from_file(essay_files[sentence_essay_path][i])\n",
    "    gpt_rep_essay = read_essay_from_file(essay_files[gpt_rep_essay_path][i])\n",
    "    word_substituted_essay = read_essay_from_file(word_sub_essay_files[i])\n",
    "\n",
    "   \n",
    "    similarity_gpt = compute_similarity(student_essay, gpt_essay)\n",
    "    similarity_rep= compute_similarity(student_essay, gpt_rep_essay) \n",
    "    similarity_gs = compute_similarity(gpt_essay, gpt_rep_essay)\n",
    "    similarity_ss = compute_similarity(gpt_essay, sentence_substituted_essay)\n",
    "    similarity_ws = compute_similarity(gpt_essay, word_substituted_essay)\n",
    "\n",
    "    quality_score_student = assess_quality( student_essay)\n",
    "    quality_score_gpt = assess_quality( gpt_essay)\n",
    "    quality_score_rep = assess_quality(gpt_rep_essay)\n",
    "    quality_score_ss = assess_quality(sentence_substituted_essay)\n",
    "    quality_score_ws = assess_quality(word_substituted_essay)\n",
    "\n",
    "    # Add results for this essay pair to the list\n",
    "    similarity_quality_results.append([\n",
    "        os.path.basename(essay_files[student_essay_path][i]),\n",
    "        similarity_gpt,\n",
    "        similarity_rep,\n",
    "        similarity_gs,\n",
    "        similarity_ss,\n",
    "        similarity_ws,\n",
    "        quality_score_student/10,\n",
    "        quality_score_gpt / 10,\n",
    "        quality_score_rep / 10,\n",
    "        quality_score_ss / 10,\n",
    "        quality_score_ws / 10,\n",
    "         \n",
    "    ])\n",
    "    print(similarity_quality_results)\n",
    "\n",
    "# Define DataFrame columns and convert results to a DataFrame, then save to CSV\n",
    "columns = [\n",
    "'Essay', 'Simhash gpt', 'Simhash Rep',  'Simhash Sent', 'Similarity GS', 'Simhash Word', \n",
    "'Quality Score Student','Quality gpt', 'Quality Rep', 'Quality Sent', 'Quality Word']\n",
    "    \n",
    "similarity_df = pd.DataFrame(similarity_quality_results, columns=columns)\n",
    "similarity_df.to_csv(\"Similarity_and_quality_results.csv\")\n",
    "similarity_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Essay</th>\n",
       "      <th>Simhash gpt</th>\n",
       "      <th>Simhash Rep</th>\n",
       "      <th>Simhash Sent</th>\n",
       "      <th>Simhash Word</th>\n",
       "      <th>Quality gpt</th>\n",
       "      <th>Quality Rep</th>\n",
       "      <th>Quality Sent</th>\n",
       "      <th>Quality Word</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.txt</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>22</td>\n",
       "      <td>26</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.txt</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100.txt</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.txt</td>\n",
       "      <td>27</td>\n",
       "      <td>27</td>\n",
       "      <td>26</td>\n",
       "      <td>22</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>102.txt</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>22</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>103.txt</td>\n",
       "      <td>16</td>\n",
       "      <td>16</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>104.txt</td>\n",
       "      <td>24</td>\n",
       "      <td>24</td>\n",
       "      <td>26</td>\n",
       "      <td>23</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>105.txt</td>\n",
       "      <td>13</td>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>18</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>106.txt</td>\n",
       "      <td>22</td>\n",
       "      <td>22</td>\n",
       "      <td>24</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>107.txt</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>19</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>108.txt</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>15</td>\n",
       "      <td>11</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>109.txt</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>25</td>\n",
       "      <td>23</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>11.txt</td>\n",
       "      <td>23</td>\n",
       "      <td>23</td>\n",
       "      <td>24</td>\n",
       "      <td>20</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>110.txt</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>21</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.7</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>111.txt</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>24</td>\n",
       "      <td>23</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>112.txt</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>21</td>\n",
       "      <td>23</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>113.txt</td>\n",
       "      <td>17</td>\n",
       "      <td>17</td>\n",
       "      <td>19</td>\n",
       "      <td>15</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>114.txt</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>15</td>\n",
       "      <td>14</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>115.txt</td>\n",
       "      <td>20</td>\n",
       "      <td>20</td>\n",
       "      <td>18</td>\n",
       "      <td>19</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>116.txt</td>\n",
       "      <td>12</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "      <td>12</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>117.txt</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.9</td>\n",
       "      <td>0.8</td>\n",
       "      <td>0.6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Essay  Simhash gpt  Simhash Rep  Simhash Sent  Simhash Word  \\\n",
       "0     1.txt           23           23            22            26   \n",
       "1    10.txt           19           19            19            20   \n",
       "2   100.txt           13           13            14            14   \n",
       "3   101.txt           27           27            26            22   \n",
       "4   102.txt           19           19            20            22   \n",
       "5   103.txt           16           16            19            20   \n",
       "6   104.txt           24           24            26            23   \n",
       "7   105.txt           13           13            16            18   \n",
       "8   106.txt           22           22            24            20   \n",
       "9   107.txt           19           19            19            20   \n",
       "10  108.txt           17           17            15            11   \n",
       "11  109.txt           25           25            25            23   \n",
       "12   11.txt           23           23            24            20   \n",
       "13  110.txt           17           17            17            21   \n",
       "14  111.txt           20           20            24            23   \n",
       "15  112.txt           20           20            21            23   \n",
       "16  113.txt           17           17            19            15   \n",
       "17  114.txt           15           15            15            14   \n",
       "18  115.txt           20           20            18            19   \n",
       "19  116.txt           12           12            13            12   \n",
       "20  117.txt           11           11            10            10   \n",
       "\n",
       "    Quality gpt  Quality Rep  Quality Sent  Quality Word  \n",
       "0           0.8          0.8           0.8           0.4  \n",
       "1           0.8          0.8           0.8           0.4  \n",
       "2           0.8          0.8           0.8           0.6  \n",
       "3           0.9          0.9           0.8           0.6  \n",
       "4           0.9          0.9           0.8           0.7  \n",
       "5           0.9          0.9           0.7           0.4  \n",
       "6           0.9          0.9           0.8           0.4  \n",
       "7           0.8          0.8           0.8           0.4  \n",
       "8           0.8          0.8           0.8           0.4  \n",
       "9           0.8          0.8           0.7           0.4  \n",
       "10          0.9          0.9           0.8           0.6  \n",
       "11          0.9          0.9           0.8           0.3  \n",
       "12          0.8          0.8           0.8           0.5  \n",
       "13          0.8          0.8           0.7           0.4  \n",
       "14          0.8          0.8           0.8           0.4  \n",
       "15          0.8          0.8           0.8           0.4  \n",
       "16          0.8          0.8           0.8           0.7  \n",
       "17          0.8          0.8           0.8           0.4  \n",
       "18          0.9          0.9           0.8           0.4  \n",
       "19          0.8          0.8           0.8           0.6  \n",
       "20          0.9          0.9           0.8           0.6  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define DataFrame columns and convert results to a DataFrame, then save to CSV\n",
    "columns = [\n",
    "'Essay', 'Simhash gpt', 'Simhash Rep',  'Simhash Sent', 'Simhash Word', \n",
    "'Quality gpt', 'Quality Rep', 'Quality Sent', 'Quality Word']\n",
    "    \n",
    "similarity_df = pd.DataFrame(similarity_quality_results, columns=columns)\n",
    "similarity_df.to_csv(\"Similarity_and_quality_results.csv\")\n",
    "similarity_df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17], ['111.txt', 0.1503, 0.904, 0.7724, 0.6936, 0.0455]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17], ['111.txt', 0.1503, 0.904, 0.7724, 0.6936, 0.0455], ['112.txt', 0.0, 0.8584999999999999, 0.6052000000000001, 0.5649000000000001, 0.0]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17], ['111.txt', 0.1503, 0.904, 0.7724, 0.6936, 0.0455], ['112.txt', 0.0, 0.8584999999999999, 0.6052000000000001, 0.5649000000000001, 0.0], ['113.txt', 0.0, 0.9551999999999999, 0.8559, 0.8895000000000001, 0.0]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17], ['111.txt', 0.1503, 0.904, 0.7724, 0.6936, 0.0455], ['112.txt', 0.0, 0.8584999999999999, 0.6052000000000001, 0.5649000000000001, 0.0], ['113.txt', 0.0, 0.9551999999999999, 0.8559, 0.8895000000000001, 0.0], ['114.txt', 0.0, 0.6378, 0.08220000000000001, 0.0, 0.0]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17], ['111.txt', 0.1503, 0.904, 0.7724, 0.6936, 0.0455], ['112.txt', 0.0, 0.8584999999999999, 0.6052000000000001, 0.5649000000000001, 0.0], ['113.txt', 0.0, 0.9551999999999999, 0.8559, 0.8895000000000001, 0.0], ['114.txt', 0.0, 0.6378, 0.08220000000000001, 0.0, 0.0], ['115.txt', 0.0684, 0.9066, 0.8525, 0.789, 0.22390000000000002]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17], ['111.txt', 0.1503, 0.904, 0.7724, 0.6936, 0.0455], ['112.txt', 0.0, 0.8584999999999999, 0.6052000000000001, 0.5649000000000001, 0.0], ['113.txt', 0.0, 0.9551999999999999, 0.8559, 0.8895000000000001, 0.0], ['114.txt', 0.0, 0.6378, 0.08220000000000001, 0.0, 0.0], ['115.txt', 0.0684, 0.9066, 0.8525, 0.789, 0.22390000000000002], ['116.txt', 0.0, 1.0, 0.9692000000000001, 0.9690000000000001, 0.3235]]\n",
      "[['1.txt', 0.0901, 0.9226000000000001, 0.5115999999999999, 0.3497, 0.0968], ['10.txt', 0.061799999999999994, 1.0, 1.0, 0.9427, 0.17920000000000003], ['100.txt', 0.0, 0.963, 0.9027, 0.4515, 0.11960000000000001], ['101.txt', 0.0, 0.9784999999999999, 0.9786, 0.9838, 0.0], ['102.txt', 0.0, 0.9322, 0.9320999999999999, 0.9434, 0.078], ['103.txt', 0.09910000000000001, 0.9745, 0.9776, 0.9703, 0.3881], ['104.txt', 0.0545, 0.9890000000000001, 0.9523, 0.9403, 0.0], ['105.txt', 0.0882, 1.0, 0.9179999999999999, 0.9427, 0.1631], ['106.txt', 0.0, 0.9334, 0.6087, 0.26530000000000004, 0.0742], ['107.txt', 0.8481000000000001, 0.9787, 0.9783, 0.9928, 0.0], ['108.txt', 0.21059999999999998, 1.0, 1.0, 1.0, 0.2929], ['109.txt', 0.13019999999999998, 0.9806, 0.9253, 0.6541, 0.1865], ['11.txt', 0.0913, 0.9063, 0.8076000000000001, 0.2372, 0.0], ['110.txt', 0.2235, 0.8615, 0.7136, 0.9555, 0.17], ['111.txt', 0.1503, 0.904, 0.7724, 0.6936, 0.0455], ['112.txt', 0.0, 0.8584999999999999, 0.6052000000000001, 0.5649000000000001, 0.0], ['113.txt', 0.0, 0.9551999999999999, 0.8559, 0.8895000000000001, 0.0], ['114.txt', 0.0, 0.6378, 0.08220000000000001, 0.0, 0.0], ['115.txt', 0.0684, 0.9066, 0.8525, 0.789, 0.22390000000000002], ['116.txt', 0.0, 1.0, 0.9692000000000001, 0.9690000000000001, 0.3235], ['117.txt', 0.0, 0.9756, 0.4768, 0.3579, 0.0]]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Essay</th>\n",
       "      <th>Human ACC</th>\n",
       "      <th>GPT ACC</th>\n",
       "      <th>Rephrased ACC</th>\n",
       "      <th>Sentence Sub ACC</th>\n",
       "      <th>Word Sub ACC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.txt</td>\n",
       "      <td>0.0901</td>\n",
       "      <td>0.9226</td>\n",
       "      <td>0.5116</td>\n",
       "      <td>0.3497</td>\n",
       "      <td>0.0968</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>10.txt</td>\n",
       "      <td>0.0618</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.9427</td>\n",
       "      <td>0.1792</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.9630</td>\n",
       "      <td>0.9027</td>\n",
       "      <td>0.4515</td>\n",
       "      <td>0.1196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>101.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.9785</td>\n",
       "      <td>0.9786</td>\n",
       "      <td>0.9838</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>102.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.9322</td>\n",
       "      <td>0.9321</td>\n",
       "      <td>0.9434</td>\n",
       "      <td>0.0780</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>103.txt</td>\n",
       "      <td>0.0991</td>\n",
       "      <td>0.9745</td>\n",
       "      <td>0.9776</td>\n",
       "      <td>0.9703</td>\n",
       "      <td>0.3881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>104.txt</td>\n",
       "      <td>0.0545</td>\n",
       "      <td>0.9890</td>\n",
       "      <td>0.9523</td>\n",
       "      <td>0.9403</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>105.txt</td>\n",
       "      <td>0.0882</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.9180</td>\n",
       "      <td>0.9427</td>\n",
       "      <td>0.1631</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>106.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.9334</td>\n",
       "      <td>0.6087</td>\n",
       "      <td>0.2653</td>\n",
       "      <td>0.0742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>107.txt</td>\n",
       "      <td>0.8481</td>\n",
       "      <td>0.9787</td>\n",
       "      <td>0.9783</td>\n",
       "      <td>0.9928</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>108.txt</td>\n",
       "      <td>0.2106</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.2929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>109.txt</td>\n",
       "      <td>0.1302</td>\n",
       "      <td>0.9806</td>\n",
       "      <td>0.9253</td>\n",
       "      <td>0.6541</td>\n",
       "      <td>0.1865</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>11.txt</td>\n",
       "      <td>0.0913</td>\n",
       "      <td>0.9063</td>\n",
       "      <td>0.8076</td>\n",
       "      <td>0.2372</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>110.txt</td>\n",
       "      <td>0.2235</td>\n",
       "      <td>0.8615</td>\n",
       "      <td>0.7136</td>\n",
       "      <td>0.9555</td>\n",
       "      <td>0.1700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>111.txt</td>\n",
       "      <td>0.1503</td>\n",
       "      <td>0.9040</td>\n",
       "      <td>0.7724</td>\n",
       "      <td>0.6936</td>\n",
       "      <td>0.0455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>112.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.8585</td>\n",
       "      <td>0.6052</td>\n",
       "      <td>0.5649</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>113.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.9552</td>\n",
       "      <td>0.8559</td>\n",
       "      <td>0.8895</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>114.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.6378</td>\n",
       "      <td>0.0822</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>115.txt</td>\n",
       "      <td>0.0684</td>\n",
       "      <td>0.9066</td>\n",
       "      <td>0.8525</td>\n",
       "      <td>0.7890</td>\n",
       "      <td>0.2239</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>116.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>1.0000</td>\n",
       "      <td>0.9692</td>\n",
       "      <td>0.9690</td>\n",
       "      <td>0.3235</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>117.txt</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>0.9756</td>\n",
       "      <td>0.4768</td>\n",
       "      <td>0.3579</td>\n",
       "      <td>0.0000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Essay  Human ACC  GPT ACC  Rephrased ACC  Sentence Sub ACC  Word Sub ACC\n",
       "0     1.txt     0.0901   0.9226         0.5116            0.3497        0.0968\n",
       "1    10.txt     0.0618   1.0000         1.0000            0.9427        0.1792\n",
       "2   100.txt     0.0000   0.9630         0.9027            0.4515        0.1196\n",
       "3   101.txt     0.0000   0.9785         0.9786            0.9838        0.0000\n",
       "4   102.txt     0.0000   0.9322         0.9321            0.9434        0.0780\n",
       "5   103.txt     0.0991   0.9745         0.9776            0.9703        0.3881\n",
       "6   104.txt     0.0545   0.9890         0.9523            0.9403        0.0000\n",
       "7   105.txt     0.0882   1.0000         0.9180            0.9427        0.1631\n",
       "8   106.txt     0.0000   0.9334         0.6087            0.2653        0.0742\n",
       "9   107.txt     0.8481   0.9787         0.9783            0.9928        0.0000\n",
       "10  108.txt     0.2106   1.0000         1.0000            1.0000        0.2929\n",
       "11  109.txt     0.1302   0.9806         0.9253            0.6541        0.1865\n",
       "12   11.txt     0.0913   0.9063         0.8076            0.2372        0.0000\n",
       "13  110.txt     0.2235   0.8615         0.7136            0.9555        0.1700\n",
       "14  111.txt     0.1503   0.9040         0.7724            0.6936        0.0455\n",
       "15  112.txt     0.0000   0.8585         0.6052            0.5649        0.0000\n",
       "16  113.txt     0.0000   0.9552         0.8559            0.8895        0.0000\n",
       "17  114.txt     0.0000   0.6378         0.0822            0.0000        0.0000\n",
       "18  115.txt     0.0684   0.9066         0.8525            0.7890        0.2239\n",
       "19  116.txt     0.0000   1.0000         0.9692            0.9690        0.3235\n",
       "20  117.txt     0.0000   0.9756         0.4768            0.3579        0.0000"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Define the directories for different types of essays\n",
    "student_essay_path = \"human\"\n",
    "gpt_essay_path = \"gpt_essays\"\n",
    "word_sub_essay_path = \"word_sub_essays\"\n",
    "sentence_essay_path = \"sentence_sub_essays\"\n",
    "gpt_rep_essay_path = \"gpt_rep_essays\"\n",
    "\n",
    "# Get the list of filenames from the word substitution essays directory\n",
    "word_subs_essay_filenames = sorted(os.listdir(word_sub_essay_path))\n",
    "\n",
    "# Filter essay files from each directory to include only those present in the word-substituted essays\n",
    "directories = [student_essay_path, gpt_essay_path, sentence_essay_path, gpt_rep_essay_path]\n",
    "essay_files = {dir_path: sorted([os.path.join(dir_path, f) for f in os.listdir(dir_path) if f in word_subs_essay_filenames]) for dir_path in directories}\n",
    "\n",
    "# Calculate the maximum number of essays to process based on the smallest category of available files\n",
    "max_essays = min(len(files) for files in essay_files.values())\n",
    "max_essays = min(21, max_essays)  # Ensure not to exceed 22 essays\n",
    "\n",
    "# Limit the number of essays processed to max_essays and prepare for processing\n",
    "essay_files = {dir_path: files[:max_essays] for dir_path, files in essay_files.items()}\n",
    "word_sub_essay_files = [os.path.join(word_sub_essay_path, f) for f in word_subs_essay_filenames][:max_essays]\n",
    "\n",
    "# Data container for the results\n",
    "similarity_quality_results = []\n",
    "\n",
    "# Process each pair of essays\n",
    "for i in range(max_essays):\n",
    "    student_essay = read_essay_from_file(essay_files[student_essay_path][i])\n",
    "    gpt_essay = read_essay_from_file(essay_files[gpt_essay_path][i])\n",
    "    sentence_substituted_essay = read_essay_from_file(essay_files[sentence_essay_path][i])\n",
    "    gpt_rep_essay = read_essay_from_file(essay_files[gpt_rep_essay_path][i])\n",
    "    word_substituted_essay = read_essay_from_file(word_sub_essay_files[i])\n",
    "\n",
    "    # Assuming these functions are defined elsewhere in the code: \n",
    "    human_detection_metrics = evaluate_zerogpt(student_essay)\n",
    "    gpt_detection_metrics = evaluate_zerogpt(gpt_essay)\n",
    "    rep_detection_metrics = evaluate_zerogpt( gpt_rep_essay)\n",
    "    ss_detection_metrics = evaluate_zerogpt(sentence_substituted_essay)\n",
    "    ws_detection_metrics = evaluate_zerogpt(word_substituted_essay)\n",
    "\n",
    "    # Add results for this essay pair to the list\n",
    "    similarity_quality_results.append([\n",
    "        os.path.basename(essay_files[student_essay_path][i]),\n",
    "        human_detection_metrics,\n",
    "        gpt_detection_metrics, \n",
    "        rep_detection_metrics, \n",
    "        ss_detection_metrics,\n",
    "        ws_detection_metrics, \n",
    "    ])\n",
    "    print(similarity_quality_results)\n",
    "\n",
    "# Define DataFrame columns and convert results to a DataFrame, then save to CSV\n",
    "columns = [\n",
    "    'Essay', 'Human ACC', 'GPT ACC', 'Rephrased ACC', 'Sentence Sub ACC', 'Word Sub ACC',\n",
    "]\n",
    "zero_similarity_df = pd.DataFrame(similarity_quality_results, columns=columns)\n",
    "zero_similarity_df.to_csv(\"zerogpt_manipulation_results.csv\")\n",
    "zero_similarity_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "gpt0",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}