prep_json.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set theme\n",
    "!jt -t oceans16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style>.container { width:100% !important; }</style>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# this is to make the Jupyter notebook as wide as the screen (on the Mac at least)\n",
    "from IPython.core.display import display, HTML\n",
    "display(HTML(\"<style>.container { width:100% !important; }</style>\"))\n",
    "%config InlineBackend.figure_format ='retina'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Import needed modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import BBP_RTQC\n",
    "from BBP_RTQC_paths import *\n",
    "from BBP_RTQC_global_vars import *\n",
    "import subprocess\n",
    "import numpy as np\n",
    "import glob\n",
    "\n",
    "import json #json manager\n",
    "import re #regular expressions\n",
    "from datetime import date\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# remove exising json file\n",
    "fn_out = \"BBP_RTQC_example_tests.json\"\n",
    "if os.path.exists(fn_out):\n",
    "  os.remove(fn_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "code_folding": [
     1,
     15
    ]
   },
   "outputs": [],
   "source": [
    "# these are the test functions, codes, and example files used to generate the json file\n",
    "test_functions = {   \"A\": \"BBP_Global_range_test\",\n",
    "                     \"A2\": \"BBP_Global_range_test\",\n",
    "                     \"B\": \"BBP_Noisy_profile_test\",\n",
    "                     \"C\": \"BBP_High_Deep_Values_test\",\n",
    "#                      \"D\": \"BBP_Surface_hook_test\", \n",
    "                     \"E\": \"BBP_Missing_Data_test\",\n",
    "#                      \"F\": \"BBP_Negative_nonsurface_test\",\n",
    "                     \"G\": \"BBP_Parking_hook_test\",     \n",
    "#                      \"H\": \"BBP_Stuck_Value_test\"      \n",
    "        }\n",
    "\n",
    "# for each test a list of lists is given to add multiple examples. the first element of the list is the profile \n",
    "# used to extract the input data, the second element is a description of what this example is about and becomes \n",
    "# the \"specifics\" field in the final json file\n",
    "egs = {  \"A\": [\n",
    "                [\"coriolis_BD6901654_056.nc\", \"Negative BBP only at PRES < 5 dbar\"]\n",
    "            ],\n",
    "         \"A2\": [\n",
    "                 [\"bodc_BR6901151_007.nc\", \"Negative BBP at PRES >= 5 dbar\"]\n",
    "             ],\n",
    "         \"B\": [\n",
    "                [\"bodc_BR6901183_130.nc\", \"eg 1\"], \n",
    "                [\"aoml_BD5905108_073.nc\", \"eg 2\"]\n",
    "             ],\n",
    "         \"C\": [\n",
    "                 [\"coriolis_BR6902827_363.nc\", \"eg 1\"]\n",
    "            ],\n",
    "#          \"D\": [\n",
    "#                  [\"coriolis_BD6901482_097.nc\", \"eg 1\"]\n",
    "#             ], \n",
    "         \"E\": [\n",
    "                   [\"coriolis_BR6902827_005.nc\", \"shallow profile with high deep values\"], \n",
    "                   [\"csiro_BD1901338_006.nc\", \"shallow profile with high deep values\"], \n",
    "                   [\"aoml_BD5906036_014.nc\", \"shallow profile due to missing data\"],\n",
    "                   [\"coriolis_BR7900560_060.nc\", \"only one valid bin, but with data below 1000 dbars\"], \n",
    "                   [\"coriolis_BD7900591_077.nc\", \"only one bin\"],\n",
    "                   [\"aoml_BD1902303_002.nc\", \"shallow profile, with good data below 200 dbar\"], \n",
    "                   [\"coriolis_BR7900586_001.nc\", \"shallow profile, no data deeper than 200 dbar\"],\n",
    "                   [\"coriolis_BD6901580_120.nc\", \"missing data in some bins\"],\n",
    "                   [\"csiro_BD5903955_554.nc\", \"no BBP data below 200 dbar\"],\n",
    "                   [\"coriolis_BR6901772_087.nc\", \"no data at all in the upper 1000 dbar\"],\n",
    "              ],\n",
    "#          \"F\": [\n",
    "#                  [\"coriolis_BD6901527_182.nc\", \"eg 1\"]\n",
    "#              ],\n",
    "         \"G\": [\n",
    "                 [\"coriolis_BD6901580_107.nc\", \"eg 1\"]      \n",
    "             ],\n",
    "#          \"H\": [\n",
    "#                  [\"coriolis_BD6901580_107.nc\", \"eg 1\"],\n",
    "#                  [\"csiro_BR7900907_014.nc\", \"eg 2\"]\n",
    "#              ]      \n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def create_new_test_example(test, code, specs, test_function, PRES, COUNTS, BBP, BBPmf1, \n",
    "                            QC_FLAGS_IN, QC_FLAGS_OUT, maxPRES=None, PARK_PRES=None):\n",
    "    \n",
    "    new_eg_test = {\"description\": test, \"code\": code, \"specifics\": specs, \"date\": date.today().strftime(\"%Y-%m-%d\"), \"functions\": [test_function],\\\n",
    "                    \"input\": {\"PRES\": PRES, \"BBP\": BBP, \"BBPmf1\": BBPmf1, \"flags_in\": QC_FLAGS_IN}, \\\n",
    "                    \"output\": {\"flags_out\": QC_FLAGS_OUT}  \n",
    "                    }    \n",
    "    \n",
    "    if code == \"E\": # add maxPRES to inputs of E test\n",
    "        new_eg_test['input'] = {\"PRES\": PRES, \"BBP\": BBP, \"BBPmf1\": BBPmf1, \"flags_in\": QC_FLAGS_IN, \n",
    "                                \"maxPRES\": maxPRES}\n",
    "        \n",
    "    if code == \"G\": # add maxPRES and PARK_PRES to inputs of G test\n",
    "        new_eg_test['input'] = {\"PRES\": PRES, \"BBP\": BBP, \"BBPmf1\": BBPmf1, \"flags_in\": QC_FLAGS_IN,\n",
    "                               \"maxPRES\": maxPRES, \"PARK_PRES\": PARK_PRES}\n",
    "        \n",
    "    if code == \"H\": # add COUNTS to inputs of H test\n",
    "        new_eg_test['input'] = {\"PRES\": PRES, \"COUNTS\": COUNTS, \"BBP\": BBP, \"flags_in\": QC_FLAGS_IN}\n",
    "        \n",
    "        \n",
    "    new_eg_test_str = json.dumps(new_eg_test, indent=2)   \n",
    "    \n",
    "    # format the string for better presentation\n",
    "    new_eg_test_horizontal = new_eg_test_str.replace(\"\\n\", \"\")\n",
    "    new_eg_test_horizontal = new_eg_test_horizontal.replace(\"  \", \"\")\n",
    "    new_eg_test_horizontal = new_eg_test_horizontal.replace(\",\", \", \")\n",
    "\n",
    "    return new_eg_test_horizontal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialise list of strings that will contain all the example-test inputs and outputs\n",
    "all_eg_tests = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "code_folding": [
     1
    ]
   },
   "outputs": [],
   "source": [
    "# read data from example profile\n",
    "def start_rd_4_json(fn_eg):\n",
    "\n",
    "    # extract strings to find files\n",
    "    tmp = fn_eg.split(\"_\")\n",
    "    iwmo = tmp[0]+'/'+tmp[1][2:]\n",
    "    prof = tmp[-1][:3]#'080'\n",
    "    QCstatus = tmp[1][0:2]#'BD'\n",
    "    dac = iwmo.split('/')[0]\n",
    "    wmo = iwmo.split('/')[-1]\n",
    "    \n",
    "    \n",
    "    # read meta file\n",
    "    [ds_config, SENSOR_MODEL, SENSOR_MAKER, SENSOR_SERIAL_NO, PLATFORM_TYPE, miss_no_float, dark, scale, khi] = BBP_RTQC.rd_WMOmeta(iwmo, VERBOSE=False)\n",
    "\n",
    "    # list single profiles\n",
    "    fn2glob = MAIN_DIR + \"dac/\" + iwmo + \"/profiles/\" + \"B*\" + iwmo.split(\"/\")[-1] + \"*_[0-9][0-9][0-9].nc\"\n",
    "    fn_single_profiles = np.sort(glob.glob(fn2glob))\n",
    "\n",
    "    # select specific profile to QC\n",
    "    fn_p = fn_single_profiles[0]\n",
    "    fn_p = MAIN_DIR + \"dac/\" + iwmo + \"/profiles/\" + QCstatus+wmo+'_'+prof+'.nc'\n",
    "    print(fn_p)\n",
    "    # read BBP and PRES + other data from profile file\n",
    "    [ PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS] = BBP_RTQC.rd_BBP(fn_p, miss_no_float, ds_config)\n",
    "\n",
    "    return iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [],
   "source": [
    "def ini_flags(BBP700):\n",
    "    # initialise output array\n",
    "    BBP700_QC_1st_failed_test = dict.fromkeys(tests.keys())\n",
    "    for ikey in BBP700_QC_1st_failed_test.keys():\n",
    "        BBP700_QC_1st_failed_test[ikey] = np.full(shape=BBP700.shape, fill_value='0')\n",
    "    return BBP700_QC_1st_failed_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/6901654/profiles/BD6901654_056.nc\n"
     ]
    }
   ],
   "source": [
    "# Negative BBP Test (A)\n",
    "code = \"A\"\n",
    "test = tests[code]\n",
    "test_function = test_functions[code]\n",
    "fn_eg = egs[code]\n",
    "\n",
    "for i_eg in fn_eg:\n",
    "    # read data from specific example profile\n",
    "    [iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS] = start_rd_4_json(i_eg[0])\n",
    "\n",
    "    # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "    BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "    # GLOBAL-RANGE TEST for BBP700\n",
    "    QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_Negative_BBP_test(BBP700, PRES, \n",
    "                                                                             np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "                                                                             fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "    # create string to store results of example test\n",
    "    specs = i_eg[1]\n",
    "    tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "                                  BBP700.tolist(), BBP700mf1.tolist(), \n",
    "                                  np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist())\n",
    "\n",
    "    # add example above to list to be saved\n",
    "    all_eg_tests.append(tmp)\n",
    "\n",
    "    del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "code_folding": [
     0
    ]
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/bodc/6901151/profiles/BR6901151_007.nc\n"
     ]
    }
   ],
   "source": [
    "# Negative BBP Test (A2)\n",
    "code = \"A2\"\n",
    "test = tests[code]\n",
    "test_function = test_functions[code]\n",
    "fn_eg = egs[code]\n",
    "\n",
    "for i_eg in fn_eg:\n",
    "    # read data from specific example profile\n",
    "    [iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS] = start_rd_4_json(i_eg[0])\n",
    "\n",
    "    # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "    BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "    # GLOBAL-RANGE TEST for BBP700\n",
    "    QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_Negative_BBP_test(BBP700, PRES, \n",
    "                                                                             np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "                                                                             fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "    # create string to store results of example test\n",
    "    specs = i_eg[1]\n",
    "    tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "                                  BBP700.tolist(), BBP700mf1.tolist(), \n",
    "                                  np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist())\n",
    "\n",
    "    # add example above to list to be saved\n",
    "    all_eg_tests.append(tmp)\n",
    "\n",
    "    del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/bodc/6901183/profiles/BR6901183_130.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/aoml/5905108/profiles/BD5905108_073.nc\n"
     ]
    }
   ],
   "source": [
    "# Noisy profile Test (B)\n",
    "code = \"B\"\n",
    "test = tests[code]\n",
    "test_function = test_functions[code]\n",
    "fn_eg = egs[code]\n",
    "\n",
    "for i_eg in fn_eg:\n",
    "    # read data from specific example profile\n",
    "    iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS = start_rd_4_json(i_eg[0])\n",
    "    \n",
    "    # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "    BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "    # NOISY-PROFILE TEST for BBP700\n",
    "    QC_FLAGS_OUT, BBP700_QC_1st_failed_test, res = BBP_RTQC.BBP_Noisy_profile_test(BBP700, BBP700mf1, PRES,\n",
    "                                                                      np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "                                                                      fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "    # create string to store results of example test\n",
    "    specs = i_eg[1]\n",
    "    tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "                                  BBP700.tolist(), BBP700mf1.tolist(), \n",
    "                                  np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist())\n",
    "\n",
    "    # add example above to list to be saved\n",
    "    all_eg_tests.append(tmp)\n",
    "\n",
    "    del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/6902827/profiles/BR6902827_363.nc\n"
     ]
    }
   ],
   "source": [
    "# High-Deep Value Test (C)\n",
    "code = \"C\"\n",
    "test = tests[code]\n",
    "test_function = test_functions[code]\n",
    "fn_eg = egs[code]\n",
    "\n",
    "for i_eg in fn_eg:\n",
    "    # read data from specific example profile\n",
    "    iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS = start_rd_4_json(i_eg[0])\n",
    "    \n",
    "    # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "    BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "    # HIGH DEEP-VALUES TEST for BBP700\n",
    "    QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_High_Deep_Values_test(BBP700mf1, PRES, \n",
    "                                                                                 np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "                                                                                 fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "    # create string to store results of example test\n",
    "    specs = i_eg[1]    \n",
    "    tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "                                  BBP700.tolist(), BBP700mf1.tolist(), \n",
    "                                  np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist())\n",
    "\n",
    "    # add example above to list to be saved\n",
    "    all_eg_tests.append(tmp)\n",
    "\n",
    "    del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # SURFACE_HOOK TEST for BBP700\n",
    "# code = \"D\"\n",
    "# test = tests[code]\n",
    "# test_function = test_functions[code]\n",
    "# fn_eg = egs[code]\n",
    "\n",
    "# for i_eg in fn_eg:\n",
    "#     # read data from specific example profile\n",
    "#     iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS = start_rd_4_json(i_eg[0])\n",
    "    \n",
    "#     # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "#     BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "#     # SURFACE_HOOK TEST for BBP700\n",
    "#     QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_Surface_hook_test(BBP700, BBP700mf1, PRES, \n",
    "#                                                                                  np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "#                                                                                  fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "#     # create string to store results of example test\n",
    "#     specs = i_eg[1]\n",
    "#     tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "#                                   BBP700.tolist(), BBP700mf1.tolist(), \n",
    "#                                   np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist())\n",
    "\n",
    "#     # add example above to list to be saved\n",
    "#     all_eg_tests.append(tmp)\n",
    "\n",
    "#     del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/6902827/profiles/BR6902827_005.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/csiro/1901338/profiles/BD1901338_006.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/aoml/5906036/profiles/BD5906036_014.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/7900560/profiles/BR7900560_060.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/7900591/profiles/BD7900591_077.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/aoml/1902303/profiles/BD1902303_002.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/7900586/profiles/BR7900586_001.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/6901580/profiles/BD6901580_120.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/csiro/5903955/profiles/BD5903955_554.nc\n",
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/6901772/profiles/BR6901772_087.nc\n"
     ]
    }
   ],
   "source": [
    "# MISSING DATA TEST (E)\n",
    "code = \"E\"\n",
    "test = tests[code]\n",
    "test_function = test_functions[code]\n",
    "fn_eg = egs[code]\n",
    "\n",
    "for i_eg in fn_eg:\n",
    "    # read data from specific example profile\n",
    "    iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS = start_rd_4_json(i_eg[0])\n",
    "    \n",
    "    # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "    BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "    # MISSING DATA TEST for BBP700\n",
    "    QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_Missing_Data_test(BBP700, PRES, maxPRES,\n",
    "                                                                    np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "                                                                    fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "    # create string to store results of example test\n",
    "    specs = i_eg[1]\n",
    "    tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "                                  BBP700.tolist(), BBP700mf1.tolist(), \n",
    "                                  np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist(),\n",
    "                                  maxPRES.tolist())\n",
    "\n",
    "    # add example above to list to be saved\n",
    "    all_eg_tests.append(tmp)\n",
    "\n",
    "    del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # NEGATIVE-NON-SURFACE TEST for BBP700\n",
    "# code = \"F\"\n",
    "# test = tests[code]\n",
    "# test_function = test_functions[code]\n",
    "# fn_eg = egs[code]\n",
    "\n",
    "# for i_eg in fn_eg:\n",
    "#     # read data from specific example profile\n",
    "#     iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS = start_rd_4_json(i_eg[0])\n",
    "    \n",
    "#     # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "#     BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "#     # NEGATIVE-NON-SURFACE TEST for BBP700\n",
    "#     QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_Negative_nonsurface_test(BBP700, PRES,\n",
    "#                                                                                  np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "#                                                                                  fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "#     # create string to store results of example test\n",
    "#     specs = i_eg[1]\n",
    "#     tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "#                                   BBP700.tolist(), BBP700mf1.tolist(), \n",
    "#                                   np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist())\n",
    "\n",
    "#     # add example above to list to be saved\n",
    "#     all_eg_tests.append(tmp)\n",
    "\n",
    "#     del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/users/modellers/gdal/scratch/network/BBP_RTQC/Data/202112-ArgoData/dac/coriolis/6901580/profiles/BD6901580_107.nc\n"
     ]
    }
   ],
   "source": [
    "# PARKING-HOOK TEST for BBP700\n",
    "code = \"G\"\n",
    "test = tests[code]\n",
    "test_function = test_functions[code]\n",
    "fn_eg = egs[code]\n",
    "\n",
    "for i_eg in fn_eg:\n",
    "    # read data from specific example profile\n",
    "    iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS = start_rd_4_json(i_eg[0])\n",
    "    \n",
    "    # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "    BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "    # PARKING-HOOK TEST for BBP700\n",
    "    QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_Parking_hook_test(BBP700, BBP700mf1, PRES,\n",
    "                                                                              maxPRES, PARK_PRES, \n",
    "                                                                                 np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "                                                                                 fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "    # create string to store results of example test\n",
    "    specs = i_eg[1]\n",
    "    tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(),\n",
    "                                  BBP700.tolist(), BBP700mf1.tolist(), \n",
    "                                  np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist(),\n",
    "                                  maxPRES.tolist(), PARK_PRES.tolist())\n",
    "\n",
    "    # add example above to list to be saved\n",
    "    all_eg_tests.append(tmp)\n",
    "\n",
    "    del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "code_folding": [
     0
    ],
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# # STUCk-VALUE TEST for BBP700\n",
    "# code = \"H\"\n",
    "# test = tests[code]\n",
    "# test_function = test_functions[code]\n",
    "# fn_eg = egs[code]\n",
    "\n",
    "# for i_eg in fn_eg:\n",
    "#     # read data from specific example profile\n",
    "#     iwmo, fn_p, PRES, BBP700, JULD, LAT, LON, BBP700mf1, miss_no_prof, PARK_PRES, maxPRES, innan, COUNTS = start_rd_4_json(i_eg[0])\n",
    "    \n",
    "#     if \"BD6901580_107\" in i_eg[0]:\n",
    "#         # example 1: set BBP700 to a constant value\n",
    "#         i1stgood = np.where(COUNTS>0)[0][0]\n",
    "#         BBP700[:] = BBP700[i1stgood]\n",
    "#         COUNTS[:] = COUNTS[i1stgood]\n",
    "# #         print('Stuck-value test: eg 1')\n",
    "\n",
    "\n",
    "#     # create BBP700_QC_flags, BBP700_QC_1st_failed_test\n",
    "#     BBP700_QC_1st_failed_test = ini_flags(BBP700)\n",
    "\n",
    "#     # PARKING-HOOK TEST for BBP700\n",
    "#     QC_FLAGS_OUT, BBP700_QC_1st_failed_test = BBP_RTQC.BBP_Stuck_Value_test(COUNTS, BBP700, PRES,\n",
    "#                                                                                  np.ones(BBP700.shape), BBP700_QC_1st_failed_test, \n",
    "#                                                                                  fn_p, VERBOSE=False, PLOT=False, SAVEPLOT=False)\n",
    "\n",
    "#     # create string to store results of example test\n",
    "#     specs = i_eg[1]\n",
    "#     tmp = create_new_test_example(test, code, specs, test_function, PRES.tolist(), COUNTS.tolist(), BBP700.tolist(), \n",
    "#                                   BBP700mf1.tolist(), \n",
    "#                                   np.ones(BBP700.shape).tolist(), QC_FLAGS_OUT.tolist(),\n",
    "#                                   maxPRES.tolist(), PARK_PRES.tolist())\n",
    "\n",
    "#     # add example above to list to be saved\n",
    "#     all_eg_tests.append(tmp)\n",
    "\n",
    "#     del QC_FLAGS_OUT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert list to string in json format\n",
    "all_eg_tests_str = json.dumps(all_eg_tests, indent=2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save json file\n",
    "out_file = open(fn_out, \"w\")\n",
    "out_file.write(all_eg_tests_str)\n",
    "out_file.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}