Skip to content

Commit

Permalink
Recover/detect abnormal traces
Browse files Browse the repository at this point in the history
  • Loading branch information
zhanglx13 committed Oct 1, 2024
1 parent 61a37c2 commit 07c0896
Showing 1 changed file with 22 additions and 5 deletions.
27 changes: 22 additions & 5 deletions python/perf-kernels/tools/tune_gemm/process_json.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,15 @@ def parse_args():


def parse_trace(code_fullname, trace_fullname):
instr0_clk, bar1_clk, bar2_clk, bar3_clk, instr9_clk, mfma_dsRead_cnt, mfma_dsWrite_cnt = gen_all_clk(
instr0_clk, bar1_clk, bar2_clk, bar3_clk, instr9_clk, mfma_dsRead_cnt, mfma_dsWrite_cnt, incomplete = gen_all_clk(
code_fullname, trace_fullname)
if incomplete:
return 0, 0, 0, 0, 0, 0, 0, 0, 0, incomplete
pro, loop, epi, iter_clk = gen_coarse_clk(instr0_clk, bar1_clk, bar3_clk, instr9_clk)
bar1_lat, bar2_lat = gen_fine_clk(bar1_clk, bar2_clk, bar3_clk)

lat1, lat2, lat_sum, idle1, idle2 = print_loop_eff(bar1_lat, bar2_lat, mfma_dsRead_cnt, mfma_dsWrite_cnt)
return pro, loop, epi, iter_clk, lat1, lat2, lat_sum, idle1, idle2
return pro, loop, epi, iter_clk, lat1, lat2, lat_sum, idle1, idle2, incomplete


def print_list(myList):
Expand Down Expand Up @@ -107,11 +109,19 @@ def gen_all_clk(code_fullname, trace_fullname):
instrAfterBarrier3_clk = trace_list[i][0]
lastInstr_clk = trace_list[-1][0]

incomplete = False
if len(instrAfterBarrier1_clk) != len(instrAfterBarrier2_clk):
print("different length of instrAfterBarrier1_clk and instrAfterBarrier2_clk")
exit(0)
incomplete = True

len1 = len(instrAfterBarrier1_clk)
len2 = len(instrAfterBarrier2_clk)
len3 = instrAfterBarrier3_clk

if len1 == 0 or len2 == 0 or len3 == 0:
incomplete = True

return firstInstr_clk, instrAfterBarrier1_clk, instrAfterBarrier2_clk, instrAfterBarrier3_clk, lastInstr_clk, mfma_dsRead_cnt, mfma_dsWrite_cnt
return firstInstr_clk, instrAfterBarrier1_clk, instrAfterBarrier2_clk, instrAfterBarrier3_clk, lastInstr_clk, mfma_dsRead_cnt, mfma_dsWrite_cnt, incomplete


def gen_coarse_clk(instr0_clk, bar1_clk, bar3_clk, instr9_clk):
Expand Down Expand Up @@ -207,14 +217,17 @@ def main():
print("wid,prologue,loop,epilogue,iter_clk,lat1,lat2,iter_lat,idle1,idle2")
epi_total = 0
epi_1st = 0
total = 0
flag = False
cnt = 0
for wid in range(maxwid):
if wv != -1 and wid != wv:
continue
trace_filename = f"se{se}_sm{sm}_sl{sl}_wv{wid}.json"
trace_fullname = os.path.join(trace_dir, trace_filename)
pro, loop, epi, iter_clk, lat1, lat2, lat_sum, idle1, idle2 = parse_trace(code_fullname, trace_fullname)
pro, loop, epi, iter_clk, lat1, lat2, lat_sum, idle1, idle2, incomplete = parse_trace(code_fullname, trace_fullname)
if incomplete:
continue
print(f"{wid},{pro},{loop},{epi},{iter_clk:.0f},{lat1},{lat2},{lat_sum},{idle1},{idle2}")
if not flag:
epi_1st = epi
Expand All @@ -223,9 +236,13 @@ def main():
if epi > 2 * epi_1st:
continue
epi_total += epi
total += epi + pro + loop
cnt += 1

if cnt == 0:
exit(0)
print(f"averaged epilogue cycles: {epi_total / cnt:.0f}")
print(f"averaged total cycles: {total / cnt:.0f}")
print(f"global_store info (averaged for all {maxwid} waves):")
calc_global_store_cycles(code_fullname)

Expand Down

0 comments on commit 07c0896

Please sign in to comment.