ROCm symmem allreduce is slower than fbgemm car allreduce

Start with fbgemm_gpu/experimental/gen_ai/bench/comm_bench.py and rerun the provided 8-rank benchmark with --num_iters=20 and --export_csv. Compare the ROCm symm and fbgemm allreduce timings across message sizes; done means the performance gap is understood and the issue has a measured resolution.

python3 fbgemm_gpu/experimental/gen_ai/bench/comm_bench.py --num_iters=20 --export_csv

Running benchmark with 8 ranks
[{'N': 1024,
'fbgemm_1shot_bwidth': 0.21609986687366958,
'fbgemm_1shot_time': 0.009477099776268006,
'fbgemm_2shot_bwidth': 0.16966774973717594,
'fbgemm_2shot_time': 0.012070649862289428,
'nccl_bwidth': 0.07660384301398733,
'nccl_time': 0.026734951138496398,
'symm_1shot_bwidth': 0.170175283451603,
'symm_1shot_time': 0.012034650146961211,
'symm_2shot_bwidth': 0.08925419349643432,
'symm_2shot_time': 0.02294570058584213},
{'N': 2048,
'fbgemm_1shot_bwidth': 0.43514287842525456,
'fbgemm_1shot_time': 0.00941300019621849,
'fbgemm_2shot_bwidth': 0.3551871171121755,
'fbgemm_2shot_time': 0.011531949788331986,
'nccl_bwidth': 0.1554789594685982,
'nccl_time': 0.02634440064430237,
'symm_1shot_bwidth': 0.35310191623477044,
'symm_1shot_time': 0.011600050330162048,
'symm_2shot_bwidth': 0.22222825536717503,
'symm_2shot_time': 0.01843149960041046},
{'N': 3072,
'fbgemm_1shot_bwidth': 0.525390915270513,
'fbgemm_1shot_time': 0.011694149672985077,
'fbgemm_2shot_bwidth': 0.4785140100526773,
'fbgemm_2shot_time': 0.012839749455451965,
'nccl_bwidth': 0.2297084378804756,
'nccl_time': 0.026746949553489684,
'symm_1shot_bwidth': 0.35896866105898656,
'symm_1shot_time': 0.017115700244903564,
'symm_2shot_bwidth': 0.2659291259274165,
'symm_2shot_time': 0.023103900253772736},
{'N': 5120,
'fbgemm_1shot_bwidth': 0.8990539548470523,
'fbgemm_1shot_time': 0.011389750242233276,
'fbgemm_2shot_bwidth': 0.8000218486792321,
'fbgemm_2shot_time': 0.012799650430679321,
'nccl_bwidth': 0.36599335910858,
'nccl_time': 0.027978649735450743,
'symm_1shot_bwidth': 0.5619842770453108,
'symm_1shot_time': 0.01822115033864975,
'symm_2shot_bwidth': 0.4456872955518811,
'symm_2shot_time': 0.022975750267505646},
{'N': 8192,
'fbgemm_1shot_bwidth': 1.2499093444780796,
'fbgemm_1shot_time': 0.013108150660991668,
'fbgemm_2shot_bwidth': 1.4509901141387298,
'fbgemm_2shot_time': 0.0112916000187397,
'nccl_bwidth': 0.5873969757073986,
'nccl_time': 0.02789255082607269,
'symm_1shot_bwidth': 0.8803987029137805,
'symm_1shot_time': 0.018609750270843505,
'symm_2shot_bwidth': 0.8761567539624059,
'symm_2shot_time': 0.018699850142002105},
{'N': 13312,
'fbgemm_1shot_bwidth': 2.070979859785417,
'fbgemm_1shot_time': 0.01285575032234192,
'fbgemm_2shot_bwidth': 1.9050684909879292,
'fbgemm_2shot_time': 0.013975350558757782,
'nccl_bwidth': 0.6573178024127491,
'nccl_time': 0.04050399959087372,
'symm_1shot_bwidth': 1.4397927865696751,
'symm_1shot_time': 0.018491549789905547,
'symm_2shot_bwidth': 1.0910693027959597,
'symm_2shot_time': 0.024401749670505523},
{'N': 22016,
'fbgemm_1shot_bwidth': 3.2838746891899673,
'fbgemm_1shot_time': 0.01340855062007904,
'fbgemm_2shot_bwidth': 2.9239949083295835,
'fbgemm_2shot_time': 0.015058849751949311,
'nccl_bwidth': 1.0965372068502555,
'nccl_time': 0.040155500173568726,
'symm_1shot_bwidth': 2.325527266142826,
'symm_1shot_time': 0.018934200704097747,
'symm_2shot_bwidth': 1.676240917984534,
'symm_2shot_time': 0.02626830041408539},
{'N': 36864,
'fbgemm_1shot_bwidth': 5.0449560710253785,
'fbgemm_1shot_time': 0.014614200592041016,
'fbgemm_2shot_bwidth': 4.492678697797537,
'fbgemm_2shot_time': 0.01641069948673248,
'nccl_bwidth': 1.8563382621278357,
'nccl_time': 0.039716899394989014,
'symm_1shot_bwidth': 3.6662722561857475,
'symm_1shot_time': 0.02010979950428009,
'symm_2shot_bwidth': 2.576136692833097,
'symm_2shot_time': 0.028619599342346192},
{'N': 60928,
'fbgemm_1shot_bwidth': 7.1004416767862955,
'fbgemm_1shot_time': 0.017161749303340912,
'fbgemm_2shot_bwidth': 7.395611001240528,
'fbgemm_2shot_time': 0.016476799547672272,
'nccl_bwidth': 3.0440142797610417,
'nccl_time': 0.04003134965896606,
'symm_1shot_bwidth': 5.5560065810869395,
'symm_1shot_time': 0.0219322994351387,
'symm_2shot_bwidth': 3.7497672823536177,
'symm_2shot_time': 0.03249695003032684},
{'N': 101888,
'fbgemm_1shot_bwidth': 7.52567135579981,
'fbgemm_1shot_time': 0.02707745134830475,
'fbgemm_2shot_bwidth': 12.252781139887329,
'fbgemm_2shot_time': 0.016630999743938446,
'nccl_bwidth': 5.143423070261001,
'nccl_time': 0.039618751406669615,
'symm_1shot_bwidth': 5.5234324500225656,
'symm_1shot_time': 0.036893001198768614,
'symm_2shot_bwidth': 6.164625373417774,
'symm_2shot_time': 0.03305569887161255},
{'N': 169472,
'fbgemm_1shot_bwidth': 11.572596536697198,
'fbgemm_1shot_time': 0.029288500547409058,
'fbgemm_2shot_bwidth': 19.819374223474767,
'fbgemm_2shot_time': 0.017101649940013886,
'nccl_bwidth': 8.457657216412821,
'nccl_time': 0.04007540047168732,
'symm_1shot_bwidth': 8.711914555772593,
'symm_1shot_time': 0.0389057993888855,
'symm_2shot_bwidth': 10.132173903266294,
'symm_2shot_time': 0.03345224857330322},
{'N': 282112,
'fbgemm_1shot_bwidth': 14.514267350468026,
'fbgemm_1shot_time': 0.03887374997138977,
'fbgemm_2shot_bwidth': 30.182333589369023,
'fbgemm_2shot_time': 0.018693849444389343,
'nccl_bwidth': 10.027476115773338,
'nccl_time': 0.05626779794692993,
'symm_1shot_bwidth': 11.01165542448017,
'symm_1shot_time': 0.05123879909515381,
'symm_2shot_bwidth': 16.37919554493729,
'symm_2shot_time': 0.03444760143756866},
{'N': 470016,
'fbgemm_1shot_bwidth': 17.45634498163174,
'fbgemm_1shot_time': 0.05385044813156128,
'fbgemm_2shot_bwidth': 41.83674934540085,
'fbgemm_2shot_time': 0.02246904969215393,
'nccl_bwidth': 15.87574104439456,
'nccl_time': 0.0592118501663208,
'symm_1shot_bwidth': 12.78648046179834,
'symm_1shot_time': 0.07351765036582947,
'symm_2shot_bwidth': 24.260424797888245,
'symm_2shot_time': 0.038747549057006836},
{'N': 783360,
'fbgemm_1shot_bwidth': 20.50852472580541,
'fbgemm_1shot_time': 0.07639359831809997,
'fbgemm_2shot_bwidth': 53.66150443894756,
'fbgemm_2shot_time': 0.029196348786354066,
'nccl_bwidth': 25.079416611998376,
'nccl_time': 0.06247035264968872,
'symm_1shot_bwidth': 15.194860304031323,
'symm_1shot_time': 0.10310854911804199,
'symm_2shot_bwidth': 33.47785294988142,
'symm_2shot_time': 0.046798700094223024},
{'N': 1305600,
'fbgemm_1shot_bwidth': 22.18470798606954,
'fbgemm_1shot_time': 0.11770269870758057,
'fbgemm_2shot_bwidth': 68.0157616602996,
'fbgemm_2shot_time': 0.03839110136032105,
'nccl_bwidth': 38.62065020183232,
'nccl_time': 0.06761149764060974,
'symm_1shot_bwidth': 16.140472148493032,
'symm_1shot_time': 0.1617796540260315,
'symm_2shot_bwidth': 43.0650873942165,
'symm_2shot_time': 0.06063380241394043},
{'N': 2175488,
'fbgemm_1shot_bwidth': 24.031862889072666,
'fbgemm_1shot_time': 0.18105030059814453,
'fbgemm_2shot_bwidth': 82.54413751013578,
'fbgemm_2shot_time': 0.05271090269088745,
'nccl_bwidth': 53.38840309434399,
'nccl_time': 0.08149664998054504,
'symm_1shot_bwidth': 17.47448681082015,
'symm_1shot_time': 0.24899020195007324,
'symm_2shot_bwidth': 55.25550585951041,
'symm_2shot_time': 0.07874284982681275},
{'N': 3624960,
'fbgemm_1shot_bwidth': 25.906921647574368,
'fbgemm_1shot_time': 0.2798449039459229,
'fbgemm_2shot_bwidth': 95.88503412616686,
'fbgemm_2shot_time': 0.07561054825782776,
'nccl_bwidth': 72.02587967759287,
'nccl_time': 0.10065715312957764,
'symm_1shot_bwidth': 18.385170863194713,
'symm_1shot_time': 0.3943351984024048,
'symm_2shot_bwidth': 68.79806978521225,
'symm_2shot_time': 0.10537970066070557},
{'N': 6041088,
'fbgemm_1shot_bwidth': 26.79060979365397,
'fbgemm_1shot_time': 0.4509854793548584,
'fbgemm_2shot_bwidth': 103.35161348319318,
'fbgemm_2shot_time': 0.11690360307693481,
'nccl_bwidth': 94.66975802722887,
'nccl_time': 0.12762445211410522,
'symm_1shot_bwidth': 18.768079801355,
'symm_1shot_time': 0.6437619686126709,
'symm_2shot_bwidth': 81.9476150881304,
'symm_2shot_time': 0.14743779897689818},
{'N': 10067456,
'fbgemm_1shot_bwidth': 0.12319780093105741,
'fbgemm_1shot_time': 163.43564453125,
'fbgemm_2shot_bwidth': 108.34488796279796,
'fbgemm_2shot_time': 0.185840904712677,
'nccl_bwidth': 106.14290873427711,
'nccl_time': 0.18969625234603882,
'symm_1shot_bwidth': 19.017884088301123,
'symm_1shot_time': 1.0587356567382813,
'symm_2shot_bwidth': 84.9453555333993,
'symm_2shot_time': 0.23703370094299317},
{'N': 16777216,
'fbgemm_1shot_bwidth': 28.02392784267898,
'fbgemm_1shot_time': 1.1973493576049805,
'fbgemm_2shot_bwidth': 111.08473038016218,
'fbgemm_2shot_time': 0.3020616054534912,
'nccl_bwidth': 125.85747263716175,
'nccl_time': 0.26660659313201907,
'symm_1shot_bwidth': 19.2322950070967,
'symm_1shot_time': 1.744692039489746,
'symm_2shot_bwidth': 90.17912823748816,
'symm_2shot_time': 0.37208645343780516}]