[Feature] Parallel Rendering Benchmark Code (#363)

* check really just rendering performance * work * benchmark isaac lab code * benchmark isaac lab code * work * make table scene faster * w * update * update * update * work * new results on 4090 * benchmark cartpole isaac lab env * Update performance_benchmarking.md * Update cartpole.py * update benchmarking code for testing rendering * plots * catch gpu mem errors in isaac lab * update scripts * Update README.md * upload some results for 3080 * organize scripts * update figures * bug fix * bug fixes * updates * work
haosulab · Jun 5, 2024 · e0a37f4 · e0a37f4
1 parent 7033cb2
commit e0a37f4
Show file tree

Hide file tree

Showing 34 changed files with 1,178 additions and 403 deletions.
diff --git a/.gitignore b/.gitignore
@@ -37,6 +37,10 @@ imgui.ini
 /videos
 /examples/benchmarking/videos
 
+# benchmarking
+/benchmark_results
+/mani_skill/examples/benchmarking/benchmark_results
+/mani_skill/examples/benchmarking/videos
 # testing
 .coverage*
 htmlcov/
diff --git a/...uide/additional_resources/benchmarking_results/rtx_3080/fps:camera_size_rgb.png b/...uide/additional_resources/benchmarking_results/rtx_3080/fps:camera_size_rgb.png
diff --git a/...uide/additional_resources/benchmarking_results/rtx_3080/fps:num_cameras_rgb.png b/...uide/additional_resources/benchmarking_results/rtx_3080/fps:num_cameras_rgb.png
diff --git a/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x128x128_rgb.png b/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x128x128_rgb.png
diff --git a/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x160x160_rgb.png b/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x160x160_rgb.png
diff --git a/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x224x224_rgb.png b/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x224x224_rgb.png
diff --git a/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x256x256_rgb.png b/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x256x256_rgb.png
diff --git a/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x512x512_rgb.png b/...ditional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x512x512_rgb.png
diff --git a/...additional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x80x80_rgb.png b/...additional_resources/benchmarking_results/rtx_3080/fps:num_envs_1x80x80_rgb.png
diff --git a/...guide/additional_resources/benchmarking_results/rtx_3080/fps:num_envs_state.png b/...guide/additional_resources/benchmarking_results/rtx_3080/fps:num_envs_state.png
diff --git a/docs/source/user_guide/additional_resources/benchmarking_results/rtx_3080/isaac_lab.csv b/docs/source/user_guide/additional_resources/benchmarking_results/rtx_3080/isaac_lab.csv
@@ -0,0 +1,55 @@
+env.step/dt,env.step/fps,env.step/psps,env.step/total_steps,env.step/cpu_mem_use,env.step/gpu_mem_use,env.step+env.reset/dt,env.step+env.reset/fps,env.step+env.reset/psps,env.step+env.reset/total_steps,env.step+env.reset/cpu_mem_use,env.step+env.reset/gpu_mem_use,env_id,obs_mode,num_envs,num_cameras,camera_width,camera_height,sensor_settings,gpu_type
+1.813408374786377,220.57910703490649,55.14477675872661,100,5555986432,3975348224,1.7134621143341064,233.44548832084908,58.36137208021227,100,5565988864,4411752448,CartpoleBalanceBenchmark-v1,rgb,4,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+1.8865888118743896,212.0228835676103,53.00572089190258,100,5577715712,3986161664,1.9101135730743408,209.41163166344987,52.35290791586247,100,5587357696,4405723136,CartpoleBalanceBenchmark-v1,rgb,4,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+1.896440029144287,210.9215128624385,52.730378215609626,100,5570162688,3976855552,1.8055648803710933,221.5373174060568,55.38432935151421,100,5596766208,4413259776,CartpoleBalanceBenchmark-v1,rgb,4,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+1.848249912261963,216.42094899951263,54.105237249878165,100,5506600960,4000055296,1.7860469818115234,223.95827437545583,55.989568593863964,100,5526163456,4436459520,CartpoleBalanceBenchmark-v1,rgb,4,1.0,224.0,224.0,RGB(224x224),NVIDIA GeForce RTX 3080 Laptop GPU
+1.869688510894776,213.9393795646593,53.484844891164826,100,5597540352,4032495616,1.7969160079956057,222.6036154278493,55.65090385696234,100,5607501824,4468899840,CartpoleBalanceBenchmark-v1,rgb,4,1.0,256.0,256.0,RGB(256x256),NVIDIA GeForce RTX 3080 Laptop GPU
+2.1866941452026367,182.92453056480505,45.73113264120126,100,5744246784,4321050624,2.139744281768799,186.93822594041183,46.73455648510296,100,5745770496,4740677632,CartpoleBalanceBenchmark-v1,rgb,4,1.0,512.0,512.0,RGB(512x512),NVIDIA GeForce RTX 3080 Laptop GPU
+1.819949626922608,879.1452116757072,54.9465757297317,100,5594279936,3993632768,1.7853028774261477,896.2064757923335,56.01290473702085,100,5603536896,4295819264,CartpoleBalanceBenchmark-v1,rgb,16,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+1.94730806350708,821.6470880926862,51.35294300579289,100,5556740096,4032495616,1.953331470489502,819.1134091537687,51.19458807211054,100,5575741440,4452122624,CartpoleBalanceBenchmark-v1,rgb,16,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+1.9883956909179688,804.6688128062374,50.29180080038984,100,5563273216,4086169600,2.010262012481689,795.9161492709019,49.74475932943137,100,5579788288,4505796608,CartpoleBalanceBenchmark-v1,rgb,16,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+2.2578940391540527,708.6249275893653,44.28905797433533,100,5654761472,4225564672,2.327617883682251,687.3980524109171,42.96237827568232,100,5669359616,4645191680,CartpoleBalanceBenchmark-v1,rgb,16,1.0,224.0,224.0,RGB(224x224),NVIDIA GeForce RTX 3080 Laptop GPU
+2.384113550186157,671.1089746019304,41.94431091262065,100,5718048768,4354605056,2.49798583984375,640.5160407554915,40.03225254721822,100,5721120768,4791009280,CartpoleBalanceBenchmark-v1,rgb,16,1.0,256.0,256.0,RGB(256x256),NVIDIA GeForce RTX 3080 Laptop GPU
+3.4059674739837646,469.7637344518066,29.36023340323792,100,6091051008,5546573824,3.4346725940704346,465.837705393584,29.114856587099,100,6092595200,5966004224,CartpoleBalanceBenchmark-v1,rgb,16,1.0,512.0,512.0,RGB(512x512),NVIDIA GeForce RTX 3080 Laptop GPU
+1.9805328845977783,1615.7267697425182,50.49146155445369,100,5584904192,4009492480,1.8713221549987795,1710.0209023080197,53.43815319712561,100,5587050496,4445896704,CartpoleBalanceBenchmark-v1,rgb,32,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+2.162653684616089,1479.6636293471367,46.23948841709802,100,5684887552,4192927744,2.1382408142089844,1496.5573469253045,46.76741709141576,100,5704785920,4629397504,CartpoleBalanceBenchmark-v1,rgb,32,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+2.1709771156311035,1473.9906639088451,46.06220824715141,100,5697372160,4331274240,2.0756194591522217,1541.7084214979498,48.17838817181093,100,5699108864,4767678464,CartpoleBalanceBenchmark-v1,rgb,32,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+2.81467080116272,1136.900272201674,35.52813350630231,100,5817282560,4620943360,2.7599315643310547,1159.4490390110843,36.23278246909639,100,5826564096,5057282048,CartpoleBalanceBenchmark-v1,rgb,32,1.0,224.0,224.0,RGB(224x224),NVIDIA GeForce RTX 3080 Laptop GPU
+2.731447458267212,1171.5400163801908,36.61062551188096,100,5936381952,4830986240,2.543877601623535,1257.9221570871646,39.31006740897389,100,5938245632,5250547712,CartpoleBalanceBenchmark-v1,rgb,32,1.0,256.0,256.0,RGB(256x256),NVIDIA GeForce RTX 3080 Laptop GPU
+4.7348291873931885,675.8427544799762,21.120086077499256,100,6900486144,7544504320,4.746451139450073,674.1879155572122,21.068372361162883,100,6910472192,7963934720,CartpoleBalanceBenchmark-v1,rgb,32,1.0,512.0,512.0,RGB(512x512),NVIDIA GeForce RTX 3080 Laptop GPU
+2.0033938884735107,3194.5789776150764,49.91529652523557,100,5670486016,4071489536,1.9699828624725344,3248.7592262438934,50.76186291006084,100,5671952384,4491116544,CartpoleBalanceBenchmark-v1,rgb,64,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+2.1170308589935303,3023.101894245727,47.23596709758949,100,5776646144,4321050624,2.0298502445220947,3152.941955827292,49.26471805980144,100,5785853952,4740677632,CartpoleBalanceBenchmark-v1,rgb,64,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+2.3021039962768555,2780.065544541248,43.438524133457,100,5858746368,4564385792,2.279712438583374,2807.3716191929,43.86518154988905,100,5860327424,5000855552,CartpoleBalanceBenchmark-v1,rgb,64,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+2.752624988555908,2325.053367824576,36.328958872259,100,6003728384,5177212928,2.666208505630493,2400.412415789873,37.50644399671677,100,6013829120,5613486080,CartpoleBalanceBenchmark-v1,rgb,64,1.0,224.0,224.0,RGB(224x224),NVIDIA GeForce RTX 3080 Laptop GPU
+2.885458469390869,2218.0184077822,34.656537621596875,100,6186536960,5548670976,2.689035654067993,2380.03538194744,37.18805284292875,100,6196895744,5968101376,CartpoleBalanceBenchmark-v1,rgb,64,1.0,256.0,256.0,RGB(256x256),NVIDIA GeForce RTX 3080 Laptop GPU
+5.07990574836731,1259.8658945703808,19.6854046026622,100,7900925952,10415439872,5.034799337387085,1271.152943966465,19.861764749476016,100,7914299392,10834870272,CartpoleBalanceBenchmark-v1,rgb,64,1.0,512.0,512.0,RGB(512x512),NVIDIA GeForce RTX 3080 Laptop GPU
+2.066894292831421,6192.866294320929,48.38176792438226,100,5734117376,4239589376,2.0016164779663086,6394.831447933079,49.95962068697718,100,5736153088,4659216384,CartpoleBalanceBenchmark-v1,rgb,128,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+2.340632200241089,5468.608010554404,42.72350008245628,100,5927534592,4751228928,2.248762845993042,5692.01862384363,44.46889549877836,100,5935464448,5187567616,CartpoleBalanceBenchmark-v1,rgb,128,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+2.6315746307373047,4864.007978528713,38.00006233225557,100,6091214848,5233704960,2.420311450958252,5288.575565319171,41.31699660405602,100,6101336064,5653200896,CartpoleBalanceBenchmark-v1,rgb,128,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+3.0948946475982666,4135.843528610318,32.31127756726811,100,6498230272,6498484224,2.982393264770508,4291.855185967551,33.53011864037149,100,6508417024,6917914624,CartpoleBalanceBenchmark-v1,rgb,128,1.0,224.0,224.0,RGB(224x224),NVIDIA GeForce RTX 3080 Laptop GPU
+3.4508626461029053,3709.2174660892897,28.978261453822576,100,6830256128,7269580800,3.36116909980774,3808.198760583681,29.75155281706001,100,6837317632,7689011200,CartpoleBalanceBenchmark-v1,rgb,128,1.0,256.0,256.0,RGB(256x256),NVIDIA GeForce RTX 3080 Laptop GPU
+2.282992362976074,11213.353323104518,43.80216141837703,100,5897297920,4583784448,2.1928865909576416,11674.110328168128,45.60199346940675,100,5904826368,5020188672,CartpoleBalanceBenchmark-v1,rgb,256,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+2.708374261856079,9452.164850531415,36.92251894738834,100,6214520832,5532352512,2.667500734329224,9596.998295274112,37.4882745909145,100,6218686464,5968560128,CartpoleBalanceBenchmark-v1,rgb,256,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+3.0558228492736816,8377.448976168464,32.72441006315806,100,6570385408,6482034688,2.9609158039093018,8645.973643087142,33.77333454330915,100,6580404224,6901465088,CartpoleBalanceBenchmark-v1,rgb,256,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+3.8851003646850586,6589.276362767847,25.739360792061905,100,7414235136,8892317696,3.787443399429321,6759.176917035201,26.40303483216876,100,7421710336,9328525312,CartpoleBalanceBenchmark-v1,rgb,256,1.0,224.0,224.0,RGB(224x224),NVIDIA GeForce RTX 3080 Laptop GPU
+4.414162874221802,5799.514138796515,22.654352104673887,100,7920762880,10399121408,4.341148138046265,5897.05745713651,23.03538069193949,100,7933820928,10835329024,CartpoleBalanceBenchmark-v1,rgb,256,1.0,256.0,256.0,RGB(256x256),NVIDIA GeForce RTX 3080 Laptop GPU
+2.678284883499145,19116.711711828,37.337327562164056,100,6140813312,5317918720,2.582207679748535,19827.99462705728,38.72655200597125,100,6144851968,5754191872,CartpoleBalanceBenchmark-v1,rgb,512,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+3.40029239654541,15057.52859725169,29.409235541507208,100,6915297280,7340359680,3.3332629203796387,15360.324469744686,30.00063372997009,100,6921842688,7776567296,CartpoleBalanceBenchmark-v1,rgb,512,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+3.922881841659546,13051.629405778947,25.491463683162007,100,7569498112,9189654528,3.876960039138794,13206.223299472873,25.793404881782955,100,7571677184,9625862144,CartpoleBalanceBenchmark-v1,rgb,512,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+5.512701988220215,9287.64154300494,18.139924888681524,100,9364905984,14201192448,5.45893144607544,9379.124926877208,18.318603372807047,100,9373179904,14619181056,CartpoleBalanceBenchmark-v1,rgb,512,1.0,224.0,224.0,RGB(224x224),NVIDIA GeForce RTX 3080 Laptop GPU
+3.349186658859253,30574.58733425084,29.85799544360433,100,6679736320,6584795136,3.2126107215881348,31874.38780300751,31.12733183887452,100,6686322688,7021002752,CartpoleBalanceBenchmark-v1,rgb,1024,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+4.749979019165039,21557.98995886935,21.05272456920835,100,8151982080,10516561920,4.616846084594727,22179.643445702783,21.659808052444127,100,8154714112,10951852032,CartpoleBalanceBenchmark-v1,rgb,1024,1.0,128.0,128.0,RGB(128x128),NVIDIA GeForce RTX 3080 Laptop GPU
+5.767221212387085,17755.5179919336,17.339373038997657,100,9430515712,14149287936,5.630961418151856,18185.171660012693,17.758956699231145,100,9447645184,14567276544,CartpoleBalanceBenchmark-v1,rgb,1024,1.0,160.0,160.0,RGB(160x160),NVIDIA GeForce RTX 3080 Laptop GPU
+4.952956914901733,41349.03725566997,20.189959597495104,100,7822176256,9292349440,4.641236305236816,44126.173831942,21.54598331637793,100,7830769664,9727901696,CartpoleBalanceBenchmark-v1,rgb,2048,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+8.551520586013794,47897.91428087188,11.693826728728489,100,10026659840,14558167040,7.959010124206543,51463.686263476666,12.564376529169108,100,10039672832,14994374656,CartpoleBalanceBenchmark-v1,rgb,4096,1.0,80.0,80.0,RGB(80x80),NVIDIA GeForce RTX 3080 Laptop GPU
+4.4419825077056885,900.4988185030078,225.124704625752,1000,3718590464,2593980416,4.29351282119751,931.6380704050988,232.9095176012747,1000,3723440128,2593980416,CartpoleBalanceBenchmark-v1,state,4,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+4.795642375946045,3336.3622108797576,208.52263817998485,1000,3712983040,2593980416,4.444350719451904,3600.075918844944,225.00474492780896,1000,3717963776,2593980416,CartpoleBalanceBenchmark-v1,state,16,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+4.745781183242798,6742.830898523299,210.7134655788531,1000,3718029312,2593980416,4.636933326721191,6901.112814280516,215.6597754462661,1000,3722747904,2593980416,CartpoleBalanceBenchmark-v1,state,32,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+4.940881252288818,12953.15485883264,202.39304466926,1000,3728838656,2593980416,4.74545693397522,13486.5832501377,210.72786328340155,1000,3733164032,2593980416,CartpoleBalanceBenchmark-v1,state,64,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+5.05900502204895,25301.41785630382,197.6673270023736,1000,3750850560,2593980416,5.033664226531982,25428.791877957163,198.6624365465404,1000,3756093440,2593980416,CartpoleBalanceBenchmark-v1,state,128,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+5.2292585372924805,48955.31520852046,191.23170003328303,1000,3748139008,2593980416,4.920007705688477,52032.43883216172,203.25171418813173,1000,3753381888,2593980416,CartpoleBalanceBenchmark-v1,state,256,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+5.232645273208618,97847.25951546217,191.10792874113704,1000,3787280384,2661089280,5.632004499435425,90909.01828138188,177.55667633082396,1000,3792261120,2661089280,CartpoleBalanceBenchmark-v1,state,512,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+5.779689073562622,177172.16046862578,173.01968795764236,1000,3872198656,2728198144,5.7836079597473145,177052.11126459867,172.90245240683464,1000,3877048320,2728198144,CartpoleBalanceBenchmark-v1,state,1024,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+6.536907911300659,313297.97326034325,152.97752600602698,1000,4031459328,2866610176,6.377048492431641,321151.70559398935,156.8123562470651,1000,4035653632,2866610176,CartpoleBalanceBenchmark-v1,state,2048,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+7.178306579589844,570608.1169124485,139.30862229307826,1000,4367384576,3139239936,7.191975593566895,569523.6234761149,139.04385338772337,1000,4371447808,3139239936,CartpoleBalanceBenchmark-v1,state,4096,,,,,NVIDIA GeForce RTX 3080 Laptop GPU
+7.6642913818359375,1068852.890877128,130.47520640589943,1000,5082435584,3686596608,7.392808437347412,1108103.9187509832,135.26659164440713,1000,5086629888,3686596608,CartpoleBalanceBenchmark-v1,state,8192,,,,,NVIDIA GeForce RTX 3080 Laptop GPU