terminate called after throwing an instance of 'std::out_of_range'
  what():  _Map_base::at

run.sh: line 2: 225874 Aborted                 LD_PRELOAD=/trace/util/tracer_nvbit/tracer_tool/tracer_tool.so /gpu-app-collection/bin/10.1/release/conv_bench inference half 700 161 1 1 32 20 5 0 0 2 2

Processing file /....../kernel-54.trace
run.sh: line 2: 984473 Killed     

Instr 46 @ 0x2e0 (736) - NOP;
  has_guard_pred = 0
  opcode = NOP/NOP
  memop = NONE
  load/store = 0/0
Instr 47 @ 0x2f0 (752) - NOP;
  has_guard_pred = 0
  opcode = NOP/NOP
  memop = NONE
  load/store = 0/0
ASSERT FAIL: function.cpp:774:void Function::gen_new_code(std::unordered_map<std::__cxx11::basic_string<char>, Function*>&): FAIL !(nregs <= 24) MSG: instrumentation function should not use more than 24 registers!

      CUDA_SAFECALL(cuFuncGetAttribute(&binary_version,
                                       CU_FUNC_ATTRIBUTE_BINARY_VERSION, p->f));

      instrument_function_if_needed(ctx, p->f);

      nvbit_enable_instrumented(ctx, p->f, true);      // <===== Assertion error

      char buffer[1024];
      sprintf(buffer, "./traces/kernel-%d.trace", kernelid);

      if (!stop_report) {
        resultsFile = fopen(buffer, "w");

Instr 46 @ 0x2e0 (736) - NOP;
Instr 47 @ 0x2f0 (752) - NOP;

kernel 0 - kernel_info - #thread-blocks 1,  kernel instructions 28, total instructions 28
Reading data file ...

Calling job_status.py
Using logfiles ['/home/mahmood/accel-sim-framework/util/job_launching/../job_launching/logfiles/sim_log.GST.21.08.03-Tuesday.txt']
procman.id      Node                            App                     AppArgs                 Version                 Config          RunningTime     Mem     JobStatus                       Basic GPGPU-Sim Stats
-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------

$ ls -l
total 1236968
-rw-r--r-- 1 mahmood mahmood  10532923 Sep  7 16:41 kernel-1.trace
-rw-r--r-- 1 mahmood mahmood  10739066 Sep 20 23:46 kernel-1.traceg
-rw-r--r-- 1 mahmood mahmood  10532923 Sep  7 16:41 kernel-2.trace
-rw-r--r-- 1 mahmood mahmood  10739066 Sep 20 23:46 kernel-2.traceg
-rw-r--r-- 1 mahmood mahmood  10532923 Sep  7 16:41 kernel-3.trace
-rw-r--r-- 1 mahmood mahmood  10739066 Sep 20 23:46 kernel-3.traceg
-rw-r--r-- 1 mahmood mahmood  75116098 Sep  7 16:41 kernel-4.trace
-rw-r--r-- 1 mahmood mahmood  75329513 Sep 20 23:46 kernel-4.traceg
-rw-r--r-- 1 mahmood mahmood 526019061 Sep  7 16:42 kernel-5.trace
-rw-r--r-- 1 mahmood mahmood 526239748 Sep 20 23:47 kernel-5.traceg
-rw-r--r-- 1 mahmood mahmood     15283 Sep 20 23:32 kernelslist
-rw-r--r-- 1 mahmood mahmood       644 Sep 20 23:52 kernelslist.g
-rw-r--r-- 1 mahmood mahmood     90308 Sep 20 23:44 stats.csv

kernel 95 - _Z24block2D_hybrid_coarsen_xffPfS_iii - #thread-blocks 64,  kernel instructions 760284, total instructions 72987264

kernel_name = _Z24block2D_hybrid_coarsen_xffPfS_iii
kernel_launch_uid = 95
gpu_sim_cycle = 43170
gpu_sim_insn = 19821254
gpu_ipc =     459.1442
gpu_tot_sim_cycle = 4102406
gpu_tot_sim_insn = 1883019130

DEF(YYEOF,"YYEOF                     /* "end of file"  */")
DEF(YYUNDEF,"YYUNDEF                 /* "invalid token"  */")

DEF(YYEOF,"YYEOF                     /* END_OF_FILE  */")
DEF(YYUNDEF,"YYUNDEF                 /* INVALID_TOKEN  */")

     TOOL_TRACE_CORE = 0 - write the core id in the traces
----------------------------------------------------------------------------------------------------
cudaGetExportTable: UUID = 0x6e 0x16 0x3f 0xbe 0xb9 0x58 0x44 0x4d 0x83 0x5c 0xe1 0x82 0xaf 0xf1 0x99 0x1e
cudaGetExportTable: UUID = 0x35 0x77 0xf 0x1b 0x9 0x2e 0x3 0x48 0xa4 0x8e 0x5 0x6f 0xc4 0x23 0x96 0x8d
cudaGetExportTable: UUID = 0xbf 0xdb 0x43 0x2d 0xbf 0x3c 0x5a 0x4a 0x94 0x5e 0xb3 0x40 0x29 0xe8 0x1e 0x75
cudaGetExportTable: UUID = 0x21 0x31 0x8c 0x60 0x97 0x14 0x32 0x48 0x8c 0xa6 0x41 0xff 0x73 0x24 0xc8 0xf2
cudaGetExportTable: UUID = 0x42 0xd8 0x5a 0x81 0x23 0xf6 0xcb 0x47 0x82 0x98 0xf6 0xe7 0x8a 0x3a 0xec 0xdc
cudaGetExportTable: UUID = 0xb1 0x5 0x41 0xe1 0xf7 0xc7 0xc7 0x4a 0x9f 0x64 0xf2 0x23 0xbe 0x99 0xf1 0xe2
cudaGetExportTable: UUID = 0xa6 0xb1 0xff 0x99 0xec 0xc4 0xc9 0x4f 0x92 0xf9 0x19 0x28 0x66 0x3d 0x55 0x85
cudaGetExportTable: UUID = 0xf8 0x8c 0xc9 0x3e 0x53 0xfd 0x9e 0x46 0xba 0x59 0x1e 0x2b 0x87 0x3e 0xf 0x91
WARNING: this function has not been implemented yet.WARNING: this function has not been implemented yet.ERROR file nvbit_imp.cpp line 360: ▒▒U H▒▒t[▒▒▒H▒▒▒▒9▒▒▒▒▒▒f.▒

    EXCLUDE_PRED_OFF = 0 - Exclude predicated off instruction from count
----------------------------------------------------------------------------------------------------
No protocol specified
kernel 0 - kernel_info - #thread-blocks 1,  kernel instructions 28, total instructions 28
  EXIT = 1
  IMAD.MOV.U32 = 5
  MOV = 6
  STG.E = 15
  ULDC.64 = 1
Lattice spacing in x,y,z = 10.000000 10.000000 10.000000
Created orthogonal box = (0.0000000 0.0000000 -5.0000000) to (20000.000 300.00000 5.0000000)
  1 by 1 by 1 MPI processor grid
Created 60000 atoms
  create_atoms CPU = 0.007 seconds
Setting atom values ...
  57615 settings made for type/fraction
Setting atom values ...
  2385 settings made for mass
Setting atom values ...
  57615 settings made for mass

--------------------------------------------------------------------------
- Using acceleration for colloid:
-  with 1 proc(s) per device.
--------------------------------------------------------------------------
Device 0: GeForce RTX 3080, 68 CUs, 9.4/9.8 GB, 1.9 GHZ (Mixed Precision)
--------------------------------------------------------------------------

Initializing Device and compiling on process 0...Done.
Initializing Device 0 on core 0...Done.

kernel 1 - kernel_zero - #thread-blocks 1,  kernel instructions 77, total instructions 105
  EXIT = 9
  IMAD = 8
  IMAD.WIDE = 1
  ISETP.GE.AND = 8
  MOV = 17
  S2R = 8
  S2UR = 8
  SGXT = 8
  STG.E = 1
  ULDC.64 = 1
  USGXT = 8
kernel 2 - kernel_zero - #thread-blocks 1,  kernel instructions 77, total instructions 182
  EXIT = 9
  IMAD = 8
  IMAD.WIDE = 1
  ISETP.GE.AND = 8
  MOV = 17
  S2R = 8
  S2UR = 8
  SGXT = 8
  STG.E = 1
  ULDC.64 = 1
  USGXT = 8

----------------------------------------------------------------------------------------------------
gpu_ipc\s*=\s*(.*)
APPS,RTX2060
cfd-rodinia-3.1/_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M--final_kernel,1529.8308

----------------------------------------------------------------------------------------------------
gpu_occupancy\s*=\s*(.*)%
APPS,RTX2060
cfd-rodinia-3.1/_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M--final_kernel,89.0889

----------------------------------------------------------------------------------------------------
L2_BW\s*=\s*(.*)+GB\/Sec
APPS,RTX2060
cfd-rodinia-3.1/_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M--final_kernel,

----------------------------------------------------------------------------------------------------

mahmood@threadripper:accel-sim-framework$ source ./gpu-simulator/setup_environment.sh
Cloning into '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim'...
remote: Enumerating objects: 12, done.
remote: Counting objects: 100% (12/12), done.
remote: Compressing objects: 100% (11/11), done.
remote: Total 14490 (delta 1), reused 3 (delta 1), pack-reused 14478
Receiving objects: 100% (14490/14490), 33.42 MiB | 3.08 MiB/s, done.
Resolving deltas: 100% (10903/10903), done.
Note: checking out 'v4.0.0'.

You are in 'detached HEAD' state. You can look around, make experimental
changes and commit them, and you can discard any commits you make in this
state without impacting any branches by performing another checkout.

If you want to create a new branch to retain commits you create, you may
do so (now or later) by using -b with the checkout command again. Example:

  git checkout -b <new-branch-name>

HEAD is now at fa49b4a Merge pull request #194 from gpgpu-sim/dev
GPGPU-Sim version 4.0.0 (build gpgpu-sim_git-commit-fa49b4a01aaf45fd6db7fc509c6dc774eee93ac1-modified_566.0) configured with GPUWattch.

----------------------------------------------------------------------------
INFO - If you only care about PTX execution, ignore this message. GPGPU-Sim supports PTX execution in modern CUDA.
If you want to run PTXPLUS (sm_1x SASS) with a modern card configuration - set the envronment variable
$PTXAS_CUDA_INSTALL_PATH to point a CUDA version compabible with your card configurations (i.e. 8+ for PASCAL, 9+ for VOLTA etc..)
For example: "export $PTXAS_CUDA_INSTALL_PATH=/usr/local/cuda-9.1"

The following text describes why:
If you are using PTXPLUS, only sm_1x is supported and it requires that the app and simulator binaries are compiled in CUDA 4.2 or less.
The simulator requires it since CUDA headers desribe struct sizes in the exec which change from gen to gen.
The apps require 4.2 because new versions of CUDA tools have dropped parsing support for generating sm_1x
When running using modern config (i.e. volta) and PTXPLUS with CUDA 4.2, the $PTXAS_CUDA_INSTALL_PATH env variable is required to get proper register usage
(and hence occupancy) using a version of CUDA that knows the register usage on the real card.

----------------------------------------------------------------------------
setup_environment succeeded
mahmood@threadripper:accel-sim-framework$ make -j -C ./gpu-simulator/
make: Entering directory '/home/mahmood/accel-sim-framework/gpu-simulator'
if [ ! -d ./bin/release ]; then mkdir -p ./bin/release; fi;
if [ ! -d ./build/release ]; then mkdir -p ./build/release; fi;
touch ./build/release/main.makedepend
makedepend -f./build/release/main.makedepend -p./build/release/ main.cc 2> /dev/null
Makefile:85: build/release/main.makedepend: No such file or directory
make: *** [Makefile:73: depend] Error 127
make: Leaving directory '/home/mahmood/accel-sim-framework/gpu-simulator'

" | tee " + os.path.join(this_run_dir,logfile + ".nsight")

mma.sync.aligned.m8n8k4.col.row.f32.f16.f16.f32 {%f779,%f780,%f781,%f782,%f783,%f784,%f785,%f786}, {%r2139,%r2140}, {%r2131,%r2132}, {%f4106,%f4105,%f4104,%f4103,%f4102,%f4101,%f4100,%f4099};
       ^

$ ./util/hw_stats/run_hw.py -B mahmood --nsight_profiler --disable_nvprof
this_run_dir =  /home/mahmood/accel-sim-framework/util/hw_stats/../../hw_run/device-0/10.1/cfd-rodinia-3.1/_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M
exec_path =  /home/mahmood/gpu-app-collection-release/bin/10.1/release/cfd-rodinia-3.1
Running cfd-rodinia-3.1


        *** GPGPU-Sim Simulator Version 4.0.0  [build gpgpu-sim_git-commit-fa49b4a01aaf45fd6db7fc509c6dc774eee93ac1_modified_0.0] ***


GPGPU-Sim PTX: simulation mode 0 (can change with PTX_SIM_MODE_FUNC environment variable:
               1=functional simulation only, 0=detailed performance simulator)
GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)


GPGPU-Sim ** ERROR: Unknown Option: '-trace_opcode_latency_initiation_int'
==ERROR== The application returned an error code (1).
==WARNING== No kernels were profiled.
==WARNING== Profiling kernels launched by child processes requires the --target-processes all option.


        *** GPGPU-Sim Simulator Version 4.0.0  [build gpgpu-sim_git-commit-fa49b4a01aaf45fd6db7fc509c6dc774eee93ac1_modified_0.0] ***


GPGPU-Sim PTX: simulation mode 0 (can change with PTX_SIM_MODE_FUNC environment variable:
               1=functional simulation only, 0=detailed performance simulator)
GPGPU-Sim PTX: overriding embedded ptx with ptx file (PTX_SIM_USE_PTX_FILE is set)


GPGPU-Sim ** ERROR: Unknown Option: '-trace_opcode_latency_initiation_int'
==ERROR== The application returned an error code (1).
==WARNING== No kernels were profiled.
==WARNING== Profiling kernels launched by child processes requires the --target-proc

#traces format = threadblock_x threadblock_y threadblock_z warpid_tb PC mask dest_num [reg_dests] opcode src_num [reg_srcs] mem_width [adrrescompress?] [mem_addresses]

31 0 0 3 0000 ffffffff 1 R1 IMAD.MOV.U32 2 R255 R255 0
102 0 0 2 0000 ffffffff 1 R1 IMAD.MOV.U32 2 R255 R255 0

$ ./gpu-simulator/bin/release/accel-sim.out -trace ./hw_run/traces/device-0/11.2/sgemm/_i__home_mnaderan_test_input_matrix1_txt__home_mnaderan_test_input_matrix2t_txt__home_mnaderan_test_input_matrix2t_txt__o__home_mnaderan_test_output_matrix3_txt/traces/kernelslist.g -config ./gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM86_RTX3070/gpgpusim.config -config ./gpu-simulator/configs/tested-cfgs/SM86_RTX3070/trace.config
Accel-Sim [build accelsim-commit-16c3068431c7bf46d425aac134947ceb3e6a8f42_modified_3.0]

        *** GPGPU-Sim Simulator Version 4.1.0  [build gpgpu-sim_git-commit-6ad461a95ac71e0597274c4f750ce03bb3a6871e_modified_0.0] ***


GPGPU-Sim: Configuration options:

-save_embedded_ptx                      0 # saves ptx files embedded in binary as <n>.ptx
-keep                                   0 # keep intermediate files created by GPGPU-Sim when interfacing with external programs
-gpgpu_ptx_save_converted_ptxplus                    0 # Saved converted ptxplus to a file
-gpgpu_occupancy_sm_number                   86 # The SM number to pass to ptxas when getting register usage for computing GPU occupancy. This parameter is required in the config.
-ptx_opcode_latency_int           4,4,4,4,21 # Opcode latencies for integers <ADD,MAX,MUL,MAD,DIV,SHFL>Default 1,1,19,25,145,32
-ptx_opcode_latency_fp           4,4,4,4,39 # Opcode latencies for single precision floating points <ADD,MAX,MUL,MAD,DIV>Default 1,1,1,1,30
-ptx_opcode_latency_dp      64,64,64,64,330 # Opcode latencies for double precision floating points <ADD,MAX,MUL,MAD,DIV>Default 8,8,8,8,335
-ptx_opcode_latency_sfu                   21 # Opcode latencies for SFU instructionsDefault 8
-ptx_opcode_latency_tesnor                   64 # Opcode latencies for Tensor instructionsDefault 64
-ptx_opcode_initiation_int            2,2,2,2,2 # Opcode initiation intervals for integers <ADD,MAX,MUL,MAD,DIV,SHFL>Default 1,1,4,4,32,4
-ptx_opcode_initiation_fp            1,1,1,1,2 # Opcode initiation intervals for single precision floating points <ADD,MAX,MUL,MAD,DIV>Default 1,1,1,1,5
-ptx_opcode_initiation_dp      64,64,64,64,130 # Opcode initiation intervals for double precision floating points <ADD,MAX,MUL,MAD,DIV>Default 8,8,8,8,130
-ptx_opcode_initiation_sfu                    8 # Opcode initiation intervals for sfu instructionsDefault 8
-ptx_opcode_initiation_tensor                   64 # Opcode initiation intervals for tensor instructionsDefault 64
-cdp_latency         7200,8000,100,12000,1600 # CDP API latency <cudaStreamCreateWithFlags, cudaGetParameterBufferV2_init_perWarp, cudaGetParameterBufferV2_perKernel, cudaLaunchDeviceV2_init_perWarp, cudaLaunchDevicV2_perKernel>Default 7200,8000,100,12000,1600
-network_mode                           2 # Interconnection network mode
-inter_config_file                   mesh # Interconnection network config file
-icnt_in_buffer_limit                  512 # in_buffer_limit
-icnt_out_buffer_limit                  512 # out_buffer_limit
-icnt_subnets                           2 # subnets
-icnt_arbiter_algo                      1 # arbiter_algo
-icnt_verbose                           0 # inct_verbose
-icnt_grant_cycles                      1 # grant_cycles
-gpgpu_ptx_use_cuobjdump                    1 # Use cuobjdump to extract ptx and sass from binaries
-gpgpu_experimental_lib_support                    0 # Try to extract code from cuda libraries [Broken because of unknown cudaGetExportTable]
-checkpoint_option                      0 #  checkpointing flag (0 = no checkpoint)
-checkpoint_kernel                      1 #  checkpointing during execution of which kernel (1- 1st kernel)
-checkpoint_CTA                         0 #  checkpointing after # of CTA (< less than total CTA)
-resume_option                          0 #  resume flag (0 = no resume)
-resume_kernel                          0 #  Resume from which kernel (1= 1st kernel)
-resume_CTA                             0 #  resume from which CTA
-checkpoint_CTA_t                       0 #  resume from which CTA
-checkpoint_insn_Y                      0 #  resume from which CTA
-gpgpu_ptx_convert_to_ptxplus                    0 # Convert SASS (native ISA) to ptxplus and run ptxplus
-gpgpu_ptx_force_max_capability                   86 # Force maximum compute capability
-gpgpu_ptx_inst_debug_to_file                    0 # Dump executed instructions' debug information to file
-gpgpu_ptx_inst_debug_file       inst_debug.txt # Executed instructions' debug output file
-gpgpu_ptx_inst_debug_thread_uid                    1 # Thread UID for executed instructions' debug output
-gpgpu_simd_model                       1 # 1 = post-dominator
-gpgpu_shader_core_pipeline              1536:32 # shader core pipeline config, i.e., {<nthread>:<warpsize>}
-gpgpu_tex_cache:l1  N:4:128:256,L:R:m:N:L,T:512:8,128:2 # per-shader L1 texture cache  (READ-ONLY) config  {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>:<rf>}
-gpgpu_const_cache:l1 N:128:64:8,L:R:f:N:L,S:2:64,4 # per-shader L1 constant memory cache  (READ-ONLY) config  {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>}
-gpgpu_cache:il1     N:64:128:16,L:R:f:N:L,S:2:48,4 # shader L1 instruction cache config  {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>}
-gpgpu_cache:dl1     S:4:128:256,L:T:m:L:L,A:384:48,16:0,32 # per-shader L1 data cache config  {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}
-gpgpu_l1_cache_write_ratio                   25 # L1D write ratio
-gpgpu_l1_banks                         4 # The number of L1 cache banks
-gpgpu_l1_banks_byte_interleaving                   32 # l1 banks byte interleaving granularity
-gpgpu_l1_banks_hashing_function                    0 # l1 banks hashing function
-gpgpu_l1_latency                      39 # L1 Hit Latency
-gpgpu_smem_latency                    29 # smem Latency
-gpgpu_cache:dl1PrefL1                 none # per-shader L1 data cache config  {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}
-gpgpu_cache:dl1PrefShared                 none # per-shader L1 data cache config  {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq> | none}
-gpgpu_gmem_skip_L1D                    0 # global memory access skip L1D cache (implements -Xptxas -dlcm=cg, default=no skip)
-gpgpu_perfect_mem                      0 # enable perfect memory mode (no cache miss)
-n_regfile_gating_group                    4 # group of lanes that should be read/written together)
-gpgpu_clock_gated_reg_file                    0 # enable clock gated reg file for power calculations
-gpgpu_clock_gated_lanes                    0 # enable clock gated lanes for power calculations
-gpgpu_shader_registers                65536 # Number of registers per shader core. Limits number of concurrent CTAs. (default 8192)
-gpgpu_registers_per_block                65536 # Maximum number of registers per CTA. (default 8192)
-gpgpu_ignore_resources_limitation                    0 # gpgpu_ignore_resources_limitation (default 0)
-gpgpu_shader_cta                      32 # Maximum number of concurrent CTAs in shader (default 8)
-gpgpu_num_cta_barriers                   16 # Maximum number of named barriers per CTA (default 16)
-gpgpu_n_clusters                      46 # number of processing clusters
-gpgpu_n_cores_per_cluster                    1 # number of simd cores per cluster
-gpgpu_n_cluster_ejection_buffer_size                   32 # number of packets in ejection buffer
-gpgpu_n_ldst_response_buffer_size                    2 # number of response packets in ld/st unit ejection buffer
-gpgpu_shmem_per_block                49152 # Size of shared memory per thread block or CTA (default 48kB)
-gpgpu_shmem_size                  102400 # Size of shared memory per shader core (default 16kB)
-gpgpu_shmem_option      0,8,16,32,64,100 # Option list of shared memory sizes
-gpgpu_unified_l1d_size                  128 # Size of unified data cache(L1D + shared memory) in KB
-gpgpu_adaptive_cache_config                    1 # adaptive_cache_config
-gpgpu_shmem_sizeDefault               102400 # Size of shared memory per shader core (default 16kB)
-gpgpu_shmem_size_PrefL1                16384 # Size of shared memory per shader core (default 16kB)
-gpgpu_shmem_size_PrefShared                16384 # Size of shared memory per shader core (default 16kB)
-gpgpu_shmem_num_banks                   32 # Number of banks in the shared memory in each shader core (default 16)
-gpgpu_shmem_limited_broadcast                    0 # Limit shared memory to do one broadcast per cycle (default on)
-gpgpu_shmem_warp_parts                    1 # Number of portions a warp is divided into for shared memory bank conflict check
-gpgpu_mem_unit_ports                    1 # The number of memory transactions allowed per core cycle
-gpgpu_shmem_warp_parts                    1 # Number of portions a warp is divided into for shared memory bank conflict check
-gpgpu_warpdistro_shader                   -1 # Specify which shader core to collect the warp size distribution from
-gpgpu_warp_issue_shader                    0 # Specify which shader core to collect the warp issue distribution from
-gpgpu_local_mem_map                    1 # Mapping from local memory space address to simulated GPU physical address space (default = enabled)
-gpgpu_num_reg_banks                    8 # Number of register banks (default = 8)
-gpgpu_reg_bank_use_warp_id                    0 # Use warp ID in mapping registers to banks (default = off)
-gpgpu_sub_core_model                    1 # Sub Core Volta/Pascal model (default = off)
-gpgpu_enable_specialized_operand_collector                    0 # enable_specialized_operand_collector
-gpgpu_operand_collector_num_units_sp                    4 # number of collector units (default = 4)
-gpgpu_operand_collector_num_units_dp                    0 # number of collector units (default = 0)
-gpgpu_operand_collector_num_units_sfu                    4 # number of collector units (default = 4)
-gpgpu_operand_collector_num_units_int                    0 # number of collector units (default = 0)
-gpgpu_operand_collector_num_units_tensor_core                    4 # number of collector units (default = 4)
-gpgpu_operand_collector_num_units_mem                    2 # number of collector units (default = 2)
-gpgpu_operand_collector_num_units_gen                    8 # number of collector units (default = 0)
-gpgpu_operand_collector_num_in_ports_sp                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_in_ports_dp                    0 # number of collector unit in ports (default = 0)
-gpgpu_operand_collector_num_in_ports_sfu                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_in_ports_int                    0 # number of collector unit in ports (default = 0)
-gpgpu_operand_collector_num_in_ports_tensor_core                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_in_ports_mem                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_in_ports_gen                    8 # number of collector unit in ports (default = 0)
-gpgpu_operand_collector_num_out_ports_sp                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_out_ports_dp                    0 # number of collector unit in ports (default = 0)
-gpgpu_operand_collector_num_out_ports_sfu                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_out_ports_int                    0 # number of collector unit in ports (default = 0)
-gpgpu_operand_collector_num_out_ports_tensor_core                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_out_ports_mem                    1 # number of collector unit in ports (default = 1)
-gpgpu_operand_collector_num_out_ports_gen                    8 # number of collector unit in ports (default = 0)
-gpgpu_coalesce_arch                   86 # Coalescing arch (GT200 = 13, Fermi = 20)
-gpgpu_num_sched_per_core                    4 # Number of warp schedulers per core
-gpgpu_max_insn_issue_per_warp                    1 # Max number of instructions that can be issued per warp in one cycle by scheduler (either 1 or 2)
-gpgpu_dual_issue_diff_exec_units                    1 # should dual issue use two different execution unit resources (Default = 1)
-gpgpu_simt_core_sim_order                    1 # Select the simulation order of cores in a cluster (0=Fix, 1=Round-Robin)
-gpgpu_pipeline_widths 4,4,4,4,4,4,4,4,4,4,8,4,4 # Pipeline widths ID_OC_SP,ID_OC_DP,ID_OC_INT,ID_OC_SFU,ID_OC_MEM,OC_EX_SP,OC_EX_DP,OC_EX_INT,OC_EX_SFU,OC_EX_MEM,EX_WB,ID_OC_TENSOR_CORE,OC_EX_TENSOR_CORE
-gpgpu_tensor_core_avail                    1 # Tensor Core Available (default=0)
-gpgpu_num_sp_units                     4 # Number of SP units (default=1)
-gpgpu_num_dp_units                     4 # Number of DP units (default=0)
-gpgpu_num_int_units                    4 # Number of INT units (default=0)
-gpgpu_num_sfu_units                    4 # Number of SF units (default=1)
-gpgpu_num_tensor_core_units                    4 # Number of tensor_core units (default=1)
-gpgpu_num_mem_units                    1 # Number if ldst units (default=1) WARNING: not hooked up to anything
-gpgpu_scheduler                      gto # Scheduler configuration: < lrr | gto | two_level_active > If two_level_active:<num_active_warps>:<inner_prioritization>:<outer_prioritization>For complete list of prioritization values see shader.h enum scheduler_prioritization_typeDefault: gto
-gpgpu_concurrent_kernel_sm                    0 # Support concurrent kernels on a SM (default = disabled)
-gpgpu_perfect_inst_const_cache                    1 # perfect inst and const cache mode, so all inst and const hits in the cache(default = disabled)
-gpgpu_inst_fetch_throughput                    4 # the number of fetched intruction per warp each cycle
-gpgpu_reg_file_port_throughput                    2 # the number ports of the register file
-specialized_unit_1         1,4,4,4,4,BRA # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-specialized_unit_2       1,4,200,4,4,TEX # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-specialized_unit_3     1,4,32,4,4,TENSOR # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-specialized_unit_4         1,4,4,4,4,UDP # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-specialized_unit_5         0,4,4,4,4,BRA # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-specialized_unit_6         0,4,4,4,4,BRA # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-specialized_unit_7         0,4,4,4,4,BRA # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-specialized_unit_8         0,4,4,4,4,BRA # specialized unit config {<enabled>,<num_units>:<latency>:<initiation>,<ID_OC_SPEC>:<OC_EX_SPEC>,<NAME>}
-gpgpu_perf_sim_memcpy                    1 # Fill the L2 cache on memcpy
-gpgpu_simple_dram_model                    0 # simple_dram_model with fixed latency and BW
-gpgpu_dram_scheduler                    1 # 0 = fifo, 1 = FR-FCFS (defaul)
-gpgpu_dram_partition_queues          64:64:64:64 # i2$:$2d:d2$:$2i
-l2_ideal                               0 # Use a ideal L2 cache that always hit
-gpgpu_cache:dl2     S:64:128:16,L:B:m:L:P,A:192:4,32:0,32 # unified banked L2 data cache config  {<nsets>:<bsize>:<assoc>,<rep>:<wr>:<alloc>:<wr_alloc>,<mshr>:<N>:<merge>,<mq>}
-gpgpu_cache:dl2_texture_only                    0 # L2 cache used for texture only
-gpgpu_n_mem                           16 # number of memory modules (e.g. memory controllers) in gpu
-gpgpu_n_sub_partition_per_mchannel                    2 # number of memory subpartition in each memory module
-gpgpu_n_mem_per_ctrlr                    1 # number of memory chips per memory controller
-gpgpu_memlatency_stat                   14 # track and display latency statistics 0x2 enables MC, 0x4 enables queue logs
-gpgpu_frfcfs_dram_sched_queue_size                   64 # 0 = unlimited (default); # entries per chip
-gpgpu_dram_return_queue_size                  192 # 0 = unlimited (default); # entries per chip
-gpgpu_dram_buswidth                    2 # default = 4 bytes (8 bytes per cycle at DDR)
-gpgpu_dram_burst_length                   16 # Burst length of each DRAM request (default = 4 data bus cycle)
-dram_data_command_freq_ratio                    4 # Frequency ratio between DRAM data bus and command bus (default = 2 times, i.e. DDR)
-gpgpu_dram_timing_opt nbk=16:CCD=4:RRD=12:RCD=24:RAS=55:RP=24:RC=78:CL=24:WL=8:CDLR=10:WR=24:nbkgrp=4:CCDL=6:RTPL=4 # DRAM timing parameters = {nbk:tCCD:tRRD:tRCD:tRAS:tRP:tRC:CL:WL:tCDLR:tWR:nbkgrp:tCCDL:tRTPL}
-gpgpu_l2_rop_latency                  187 # ROP queue latency (default 85)
-dram_latency                         254 # DRAM latency (default 30)
-dram_dual_bus_interface                    0 # dual_bus_interface (default = 0)
-dram_bnk_indexing_policy                    0 # dram_bnk_indexing_policy (0 = normal indexing, 1 = Xoring with the higher bits) (Default = 0)
-dram_bnkgrp_indexing_policy                    1 # dram_bnkgrp_indexing_policy (0 = take higher bits, 1 = take lower bits) (Default = 0)
-dram_seperate_write_queue_enable                    0 # Seperate_Write_Queue_Enable
-dram_write_queue_size             32:28:16 # Write_Queue_Size
-dram_elimnate_rw_turnaround                    0 # elimnate_rw_turnaround i.e set tWTR and tRTW = 0
-icnt_flit_size                        40 # icnt_flit_size
-gpgpu_mem_addr_mapping dramid@8;00000000.00000000.00000000.00000000.0000RRRR.RRRRRRRR.RBBBCCCC.BCCSSSSS # mapping memory address to dram model {dramid@<start bit>;<memory address map>}
-gpgpu_mem_addr_test                    0 # run sweep test to check address mapping for aliased address
-gpgpu_mem_address_mask                    1 # 0 = old addressing mask, 1 = new addressing mask, 2 = new add. mask + flipped bank sel and chip sel bits
-gpgpu_memory_partition_indexing                    2 # 0 = no indexing, 1 = bitwise xoring, 2 = IPoly, 3 = custom indexing
-gpuwattch_xml_file         gpuwattch.xml # GPUWattch XML file
-power_simulation_enabled                    0 # Turn on power simulator (1=On, 0=Off)
-power_per_cycle_dump                    0 # Dump detailed power output each cycle
-power_trace_enabled                    0 # produce a file for the power trace (1=On, 0=Off)
-power_trace_zlevel                     6 # Compression level of the power trace output log (0=no comp, 9=highest)
-steady_power_levels_enabled                    0 # produce a file for the steady power levels (1=On, 0=Off)
-steady_state_definition                  8:4 # allowed deviation:number of samples
-gpgpu_max_cycle                        0 # terminates gpu simulation early (0 = no limit)
-gpgpu_max_insn                         0 # terminates gpu simulation early (0 = no limit)
-gpgpu_max_cta                          0 # terminates gpu simulation early (0 = no limit)
-gpgpu_max_completed_cta                    0 # terminates gpu simulation early (0 = no limit)
-gpgpu_runtime_stat                   500 # display runtime statistics such as dram utilization {<freq>:<flag>}
-liveness_message_freq                    1 # Minimum number of seconds between simulation liveness messages (0 = always print)
-gpgpu_compute_capability_major                    8 # Major compute capability version number
-gpgpu_compute_capability_minor                    6 # Minor compute capability version number
-gpgpu_flush_l1_cache                    1 # Flush L1 cache at the end of each kernel call
-gpgpu_flush_l2_cache                    0 # Flush L2 cache at the end of each kernel call
-gpgpu_deadlock_detect                    1 # Stop the simulation at deadlock (1=on (default), 0=off)
-gpgpu_ptx_instruction_classification                    0 # if enabled will classify ptx instruction types per kernel (Max 255 kernels now)
-gpgpu_ptx_sim_mode                     0 # Select between Performance (default) or Functional simulation (1)
-gpgpu_clock_domains 1132:1132:1132:3500.5 # Clock Domain Frequencies in MhZ {<Core Clock>:<ICNT Clock>:<L2 Clock>:<DRAM Clock>}
-gpgpu_max_concurrent_kernel                    8 # maximum kernels that can run concurrently on GPU
-gpgpu_cflog_interval                    0 # Interval between each snapshot in control flow logger
-visualizer_enabled                     0 # Turn on visualizer output (1=On, 0=Off)
-visualizer_outputfile                 NULL # Specifies the output log file for visualizer
-visualizer_zlevel                      6 # Compression level of the visualizer output log (0=no comp, 9=highest)
-gpgpu_stack_size_limit                 1024 # GPU thread stack size
-gpgpu_heap_size_limit              8388608 # GPU malloc heap size
-gpgpu_runtime_sync_depth_limit                    2 # GPU device runtime synchronize depth
-gpgpu_runtime_pending_launch_count_limit                 2048 # GPU device runtime pending launch count
-trace_enabled                          0 # Turn on traces
-trace_components                    none # comma seperated list of traces to enable. Complete list found in trace_streams.tup. Default none
-trace_sampling_core                    0 # The core which is printed using CORE_DPRINTF. Default 0
-trace_sampling_memory_partition                   -1 # The memory partition which is printed using MEMPART_DPRINTF. Default -1 (i.e. all)
-enable_ptx_file_line_stats                    1 # Turn on PTX source line statistic profiling. (1 = On)
-ptx_line_stats_filename gpgpu_inst_stats.txt # Output file for PTX source line statistics.
-gpgpu_kernel_launch_latency                 5000 # Kernel launch latency in cycles. Default: 0
-gpgpu_cdp_enabled                      0 # Turn on CDP
-gpgpu_TB_launch_latency                    0 # thread block launch latency in cycles. Default: 0
-trace               ./hw_run/traces/device-0/11.2/sgemm/_i__home_mnaderan_test_input_matrix1_txt__home_mnaderan_test_input_matrix2t_txt__home_mnaderan_test_input_matrix2t_txt__o__home_mnaderan_test_output_matrix3_txt/traces/kernelslist.g # traces kernel filetraces kernel file directory
-trace_opcode_latency_initiation_int                  4,2 # Opcode latencies and initiation for integers in trace driven mode <latency,initiation>
-trace_opcode_latency_initiation_sp                  4,1 # Opcode latencies and initiation for sp in trace driven mode <latency,initiation>
-trace_opcode_latency_initiation_dp                64,64 # Opcode latencies and initiation for dp in trace driven mode <latency,initiation>
-trace_opcode_latency_initiation_sfu                 21,8 # Opcode latencies and initiation for sfu in trace driven mode <latency,initiation>
-trace_opcode_latency_initiation_tensor                32,32 # Opcode latencies and initiation for tensor in trace driven mode <latency,initiation>
-trace_opcode_latency_initiation_spec_op_1                  4,4 # specialized unit config <latency,initiation>
-trace_opcode_latency_initiation_spec_op_2                200,4 # specialized unit config <latency,initiation>
-trace_opcode_latency_initiation_spec_op_3                32,32 # specialized unit config <latency,initiation>
-trace_opcode_latency_initiation_spec_op_4                  4,1 # specialized unit config <latency,initiation>
-trace_opcode_latency_initiation_spec_op_5                  4,4 # specialized unit config <latency,initiation>
-trace_opcode_latency_initiation_spec_op_6                  4,4 # specialized unit config <latency,initiation>
-trace_opcode_latency_initiation_spec_op_7                  4,4 # specialized unit config <latency,initiation>
-trace_opcode_latency_initiation_spec_op_8                  4,4 # specialized unit config <latency,initiation>
DRAM Timing Options:
nbk                                    16 # number of banks
CCD                                     4 # column to column delay
RRD                                    12 # minimal delay between activation of rows in different banks
RCD                                    24 # row to column delay
RAS                                    55 # time needed to activate row
RP                                     24 # time needed to precharge (deactivate) row
RC                                     78 # row cycle time
CDLR                                   10 # switching from write to read (changes tWTR)
WR                                     24 # last data-in to row precharge
CL                                     24 # CAS latency
WL                                      8 # Write latency
nbkgrp                                  4 # number of bank groups
CCDL                                    6 # column to column delay between accesses to different bank groups
RTPL                                    4 # read to precharge delay between accesses to different bank groups
Total number of memory sub partition = 32
addr_dec_mask[CHIP]  = 0000000000000f00         high:12 low:8
addr_dec_mask[BK]    = 0000000000070080         high:19 low:7
addr_dec_mask[ROW]   = 00000000fff80000         high:32 low:19
addr_dec_mask[COL]   = 000000000000f07f         high:16 low:0
addr_dec_mask[BURST] = 000000000000001f         high:5 low:0
sub_partition_id_mask = 0000000000000080
GPGPU-Sim uArch: clock freqs: 1132000000.000000:1132000000.000000:1132000000.000000:3500500000.000000
GPGPU-Sim uArch: clock periods: 0.00000000088339222615:0.00000000088339222615:0.00000000088339222615:0.00000000028567347522
*** Initializing Memory Statistics ***
GPGPU-Sim uArch: performance model initialization complete.
launching memcpy command : MemcpyHtoD,0x00007f806c600000,4063232
launching memcpy command : MemcpyHtoD,0x00007f806ca00000,4190208
Processing kernel ./hw_run/traces/device-0/11.2/sgemm/_i__home_mnaderan_test_input_matrix1_txt__home_mnaderan_test_input_matrix2t_txt__home_mnaderan_test_input_matrix2t_txt__o__home_mnaderan_test_output_matrix3_txt/traces/kernel-1.traceg
-kernel name = _Z9mysgemmNTPKfiS0_iPfiiff
-kernel id = 1
-grid dim = (8,66,1)
-block dim = (16,8,1)
-shmem = 512
-nregs = 52
-binary version = 86
-cuda stream id = 0
-shmem base_addr = 0x00007f8094000000
-local mem base_addr = 0x00007f8092000000
-nvbit version = 1.5.3
-accelsim tracer version = 3
launching kernel command : ./hw_run/traces/device-0/11.2/sgemm/_i__home_mnaderan_test_input_matrix1_txt__home_mnaderan_test_input_matrix2t_txt__home_mnaderan_test_input_matrix2t_txt__o__home_mnaderan_test_output_matrix3_txt/traces/kernel-1.traceg
GPGPU-Sim uArch: Shader 0 bind to kernel 1 '_Z9mysgemmNTPKfiS0_iPfiiff'
GPGPU-Sim uArch: CTA/core = 9, limited by: regs
GPGPU-Sim: Reconfigure L1 cache to 120KB
thread block = 0,0,0
Segmentation fault (core dumped)

assert(
        localaddr % 4 ==
        0);  // Address must be 4B aligned - required if accessing 4B per
             // request, otherwise access will overflow into next thread's space

$ ls hw_run/traces/device-0/10.1/cfd-rodinia-3.1/_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M/traces/
kernel-1000.trace  kernel-207.trace  kernel-340.trace  kernel-474.trace  kernel-607.trace  kernel-740.trace  kernel-874.trace
kernel-1001.trace  kernel-208.trace  kernel-341.trace  kernel-475.trace  kernel-608.trace  kernel-741.trace  kernel-875.trace
kernel-1002.trace  kernel-209.trace  kernel-342.trace  kernel-476.trace  kernel-609.trace  kernel-742.trace  kernel-876.trace
...
...

$ ./util/job_launching/run_simulations.py -B mahmood -C RTX2060 -T ./hw_run/traces/device-0/10.1/ -N cfd-rodinia-3.1
Running Simulations with GPGPU-Sim built from
accelsim-commit-21fdb684796bba6057a45eeb6429b04cc7577537_modified_1.0

Using configs: RTX2060
Benchmark: mahmood
Run Subdir = RTX2060
Parameters =
Base config file = /home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config
Traceback (most recent call last):
  File "./util/job_launching/run_simulations.py", line 363, in <module>
    config.run(version_string, benchmarks, options.run_directory, cuda_version, options.simulator_dir)
  File "./util/job_launching/run_simulations.py", line 76, in run
    self.append_gpgpusim_config(benchmark, this_run_dir, self.config_file)
  File "./util/job_launching/run_simulations.py", line 266, in append_gpgpusim_config
    config_text = open(config_text_file).read()
IOError: [Errno 2] No such file or directory: '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config'

$ ls /home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/
SM2_GTX480/       SM3_KEPLER_TITAN/ SM6_TITANX/       SM75_RTX2060/     SM7_QV100/        SM7_TITANV/

Could not fined SM75_RTX2060 in defined basenames {'QV100_SASS': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM7_QV100_SASS/gpgpusim.config', 'TESLAC2050': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/4.x-cfgs/SM2_C2050/gpgpusim.config', 'PUB_GTX480': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/GTX480/gpgpusim.config', 'TITANXX': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/TITANX-pascal/gpgpusim.config', 'GTX480': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM2_GTX480/gpgpusim.config', 'QV100': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM7_QV100/gpgpusim.config', 'QUADROFX5800': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/4.x-cfgs/SM1_QFX5800/gpgpusim.config', 'TITANK': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM3_KEPLER_TITAN/gpgpusim.config', 'RTX2060': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/Turing_RTX2060/gpgpusim.config', 'PUB_TITANX': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/SM6_TITANX/gpgpusim.config', 'QUADROFX5600': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/4.x-cfgs/SM1_QFX5600/gpgpusim.config', 'TITANX': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM6_TITANX/gpgpusim.config', 'PUB_GTX750': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/GeForceGTX750Ti/gpgpusim.config', 'QV100_old': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM7_QV100_old/gpgpusim.config', 'TITANV': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM7_TITANV/gpgpusim.config', 'PUB_GTX1080': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/GeForceGTX1080Ti/gpgpusim.config', 'TITANV_OLD': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM7_TITANV_OLD/gpgpusim.config', 'QV100_64SM': '/home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM7_QV100_SMs/gpgpusim.config'}

$ ./util/job_launching/run_simulations.py -B mahmood -C RTX2060 -T ./hw_run/traces/device-0/10.1/ -N cfd-rodinia-3.1 -l local
Running Simulations with GPGPU-Sim built from
accelsim-commit-21fdb684796bba6057a45eeb6429b04cc7577537_modified_1.0

Using configs: RTX2060
Benchmark: mahmood
Run Subdir = RTX2060
Parameters =
Base config file = /home/mahmood/accel-sim-framework/gpu-simulator/gpgpu-sim/configs/tested-cfgs/SM75_RTX2060/gpgpusim.config
Job 3 queued (cfd-rodinia-3.1-_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M RTX2060)
ProcMan spawned [pid=74837]

$ ./util/job_launching/monitor_func_test.py -v -N cfd-rodinia-3.1
Calling job_status.py

^CTraceback (most recent call last):
  File "./util/job_launching/monitor_func_test.py", line 124, in <module>
    stdout=jobstatus_out_file, stderr=jobstatus_out_file) != 0:
  File "/usr/lib/python2.7/subprocess.py", line 172, in call
    return Popen(*popenargs, **kwargs).wait()
  File "/usr/lib/python2.7/subprocess.py", line 1099, in wait
    pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0)
  File "/usr/lib/python2.7/subprocess.py", line 125, in _eintr_retry_call
    return func(*args)
KeyboardInterrupt

$ cat util/job_launching/logfiles/sim_log.cfd-rodinia-3.1.20.09.08-Tuesday.txt
15:46:14      3 cfd-rodinia-3.1        _GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M                      RTX2060                   cfd-rodinia-3.1.accelsim-commit-21fdb684796bba6057a45eeb6429b04cc7577537_modified_1.0

$ ls hw_run/traces/device-0/10.1/cfd-rodinia-3.1/_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M/traces/kernel-1.trace  -l
-rw-r--r-- 1 mahmood mahmood 10532923 Sep  7 16:41 hw_run/traces/device-0/10.1/cfd-rodinia-3.1/_GPUAPPS_ROOT_data_dirs_cuda_rodinia_3_1_cfd_rodinia_3_1_data_missile_domn_0_2M/traces/kernel-1.trace

g++ -o post-traces-processing post-traces-processing.cpp
post-traces-processing.cpp:13:23: error: ‘>>’ should be ‘> >’ within a nested template argument list
   vector<vector<string>> warp_insts_array;
                       ^
Makefile:4: recipe for target 'post-traces-processing' failed
make[1]: *** [post-traces-processing] Error 1

 ./util/plotting/plot-correlation.py -c per.kernel.stats.csv -H ./hw_run/device-0/11.0/
-----------------------------------------------------------------
All Card Summary:
HW Summary for Tesla V100-SXM2-32GB [Contains 1 Apps]:
----------------------------------------------------------------


Output Available at: file:<some-path>/accel-sim-framework/util/plotting/correl-html

gpu_tot_occupancy = 32.3427% 
max_total_param_size = 0
gpu_stall_dramfull = 4253
gpu_stall_icnt2sh    = 0
partiton_level_parallism =      24.5042
partiton_level_parallism_total  =      24.5042
partiton_level_parallism_util =      25.1316
partiton_level_parallism_util_total  =      25.1316
L2_BW  =     887.6406 GB/Sec
L2_BW_total  =     887.6406 GB/Sec
gpu_total_sim_rate=477568

========= Core cache stats =========
L1I_cache:
	L1I_total_cache_accesses = 0
	L1I_total_cache_misses = 0
	L1I_total_cache_pending_hits = 0
	L1I_total_cache_reservation_fails = 0

./util/job_launching/run_simulations.py -B sgemm -C RTX2060 -T ./hw_run/traces/device-0/10.1/ -N test_sgemm -l local
./util/hw_stats/run_hw.py -B sgemm --nsight_profiler --disable_nvprof