decapoda-research_llama-7b-hf-a100-sxm-80gb-w16a16e16-tp8-pp8-dp32-sp8-fe0.5-hbme0.9-summary.json

{
    "batch_size_per_gpu": 50,
    "max_batch_size_per_gpu": 62,
    "gradient_accumulation_steps": 2500,
    "global_batch_size": 4000000,
    "dp_size": 32,
    "tp_size": 8,
    "pp_size": 8,
    "sp_size": 8,
    "ds_zero": "NONE",
    "total_num_gpus": 2048,
    "seq_len": 2048,
    "total_num_tokens": 1000000000000.0,
    "num_params_total": 6573522944,
    "activation_recomputation": "SELECTIVE",
    "achived_flops": 156.0,
    "flops_efficiency": 0.5,
    "hbm_memory_efficiency": 0.9,
    "num_flops_total_per_micro_batch": 4478577147904000,
    "weight_memory_per_gpu": 234094592.0,
    "gradient_memory_per_gpu": 201326592.0,
    "optimizer_state_memory_per_gpu": 1610612736.0,
    "activation_memory_bs1": 1241513984.0,
    "activation_memory_per_gpu": 62075699200.0,
    "latency_per_micro_batch": 0.4485754354871795,
    "latency_fwd": 0.2097721096742479,
    "latency_fwd_attn": 0.05506368328205128,
    "latency_fwd_mlp": 0.08810189325128205,
    "latency_fwd_layernorm": 0.0009142398779358072,
    "latency_fwd_tp_comm": 0.03914683733333333,
    "latency_fwd_input_embedding": 0.005036204647594136,
    "latency_fwd_output_embedding_loss": 0.02150925128205128,
    "latency_per_iter": 1121.4385887179487,
    "total_training_latency": 136815.50782358975,
    "gpu_hours": 77832.82222853106
}