updated saving formats to work with large batch submissions

measure_perc.py

@@ -5,6 +5,7 @@
 import numpy as np
 import networkx as nx
 from multiprocessing import Pool
+import uuid
 
 def checkdir(directoryname):
     if os.path.isdir(directoryname) == False:
@@ -14,7 +15,7 @@ def checkdir(directoryname):
 
 def measure_fullnet(n,scaling, l='exp', save=False, seed=0, v=True ,remote=False):
     start = timer()
-    data=pd.DataFrame(columns = ['sticks', 'size', 'density', 'nclust', 'maxclust', 'ion', 'ioff','ioff_totaltop', 'ioff_partialtop', 'runtime', 'fname'])
+    data=pd.DataFrame(columns = ['sticks', 'size', 'density', 'nclust', 'maxclust', 'ion', 'ioff','ioff_totaltop', 'ioff_partialtop', 'runtime', 'fname','seed'])
     try:
         collection=perc.StickCollection(n,scaling=scaling,notes='run',l=l,seed=seed)
         collection.label_clusters()
@@ -69,24 +70,25 @@ def measure_fullnet(n,scaling, l='exp', save=False, seed=0, v=True ,remote=False
         ioff_partialtop=0
     end = timer()
     runtime=end - start
-    data.loc[0]=[n,scaling,n/scaling**2,nclust,maxclust,ion,ioff,ioff_totaltop,ioff_partialtop,runtime,fname]
+    data.loc[0]=[n,scaling,n/scaling**2,nclust,maxclust,ion,ioff,ioff_totaltop,ioff_partialtop,runtime,fname,seed]
     if fname:
         data.to_csv(fname+"_data.csv")
     return data
 
 def measure_async(cores,start,step,number,scaling,save=False):
+    uuid=uuid.uuid4()
     starttime = timer()
     nrange=[int(start+i*step) for i in range(number)]
     seeds=np.random.randint(low=0,high=2**32,size=number)
-    np.savetxt("seeds.csv", seeds, delimiter=",")
+    np.savetxt("seeds_{}.csv".format(uuid), seeds, delimiter=",")
     pool=Pool(cores)
     results=[pool.apply_async(measure_fullnet,args=(nrange[i],scaling,'exp',save,seeds[i])) for i in range(number)]
     output=[res.get() for res in results]
     endtime = timer()
     runtime=endtime - starttime
     print('finished with a runtime of {:.0f} seconds'.format(runtime))
     data=pd.concat(output)
-    data.to_csv('measurement_cores{}_start{}_step{}_number{}_runtime{:.0f}.csv'.format(cores, start, step, number,runtime))
+    data.to_csv('measurement_batch_{}.csv'.format(uuid))
 
 if __name__ == '__main__':
     parser = argparse.ArgumentParser()

percolation.py

@@ -26,7 +26,7 @@ def __init__(self, n=2, l='exp', pm=0.135, scaling=5, fname='', directory='data'
         else:
             self.seed=np.random.randint(low=0,high=2**32)
         np.random.seed(self.seed)
-        
+
         if not(fname):
             self.sticks, self.intersects  = self.make_intersects_kdtree( self.make_sticks(n, l=l, pm=pm, scaling=scaling))
             self.make_cnet()
@@ -177,7 +177,7 @@ def timestamp(self):
         return datetime.now().strftime('%y-%m-%d_%H%M%S_%f')
 
     def make_fname(self):
-        self.notes="{}_{}sticks_{}x{}um_{}L_{}".format( self.timestamp(),self.n,self.scaling,self.scaling,self.l,self.notes)
+        self.notes="{}_{}sticks_{}x{}um_{}L_{}".format( self.seed(),self.n,self.scaling,self.scaling,self.l,self.notes)
         fname=os.path.join(self.directory,self.notes)
         return fname
 

slurm/batchmeasure.sh

@@ -3,7 +3,6 @@
 for density in 4 9 10 11 14 20
 do
     n=$(echo $density*3600 | bc)
-    echo $n
     mkdir meas_32x$density
     echo 'python3 ~/gitrepos/networksim-cntfet/measure_perc.py -s --cores 32 --start '$n' --step 0 --number 32 --scaling 60' > meas_32x$density/measurenet.sh
     cd meas_32x$density/

slurm/subpy.sh

@@ -13,7 +13,6 @@ usage()
 
      <time> Job time in hh:mm:ss (Default $time).
     <cpunr> Request a parallel job running on <cpunr> CPUs (Default $cpunr).
-
    <memory> Requests ram in mb per processor (no suffix, Default $mem).
      <disk> Requests diskspace per processor (mb, gb or tb suffix, Default $disk).
      <node> Requests a specific node for the job.
@@ -86,8 +85,7 @@ cat > "$submitfile" <<EOF
 #SBATCH --mem-per-cpu=$mem
 #SBATCH --workdir=$PWD
 #SBATCH --kill-on-invalid-dep=yes
-#SBATCH --mail-type=END
-#SBATCH --mail-user=$USER@localhost
+
 date
 echo "User executing this job: \$SLURM_JOB_USER"
 echo -n "We are on: " && hostname