usage: bench_offline_throughput.py [-h] --model-path MODEL_PATH
0: [--remote-instance-weight-loader-seed-instance-ip REMOTE_INSTANCE_WEIGHT_LOADER_SEED_INSTANCE_IP]
0: [--remote-instance-weight-loader-seed-instance-service-port REMOTE_INSTANCE_WEIGHT_LOADER_SEED_INSTANCE_SERVICE_PORT]
0: [--remote-instance-weight-loader-send-weights-group-ports REMOTE_INSTANCE_WEIGHT_LOADER_SEND_WEIGHTS_GROUP_PORTS]
0: [--tokenizer-path TOKENIZER_PATH]
0: [--tokenizer-mode {auto,slow}]
0: [--tokenizer-worker-num TOKENIZER_WORKER_NUM]
0: [--skip-tokenizer-init]
0: [--load-format {auto,pt,safetensors,npcache,dummy,sharded_state,gguf,bitsandbytes,layered,remote,remote_instance}]
0: [--model-loader-extra-config MODEL_LOADER_EXTRA_CONFIG]
0: [--trust-remote-code]
0: [--context-length CONTEXT_LENGTH]
0: [--is-embedding] [--enable-multimodal]
0: [--revision REVISION]
0: [--model-impl MODEL_IMPL] [--host HOST]
0: [--port PORT] [--skip-server-warmup]
0: [--warmups WARMUPS] [--nccl-port NCCL_PORT]
0: [--dtype {auto,half,float16,bfloat16,float,float32}]
0: [--quantization {awq,fp8,gptq,marlin,gptq_marlin,awq_marlin,bitsandbytes,gguf,modelopt,modelopt_fp4,petit_nvfp4,w8a8_int8,w8a8_fp8,moe_wna16,qoq,w4afp8,mxfp4}]
0: [--quantization-param-path QUANTIZATION_PARAM_PATH]
0: [--kv-cache-dtype {auto,fp8_e5m2,fp8_e4m3}]
0: [--mem-fraction-static MEM_FRACTION_STATIC]
0: [--max-running-requests MAX_RUNNING_REQUESTS]
0: [--max-queued-requests MAX_QUEUED_REQUESTS]
0: [--max-total-tokens MAX_TOTAL_TOKENS]
0: [--chunked-prefill-size CHUNKED_PREFILL_SIZE]
0: [--max-prefill-tokens MAX_PREFILL_TOKENS]
0: [--schedule-policy {lpm,random,fcfs,dfs-weight,lof,priority}]
0: [--enable-priority-scheduling]
0: [--schedule-low-priority-values-first]
0: [--priority-scheduling-preemption-threshold PRIORITY_SCHEDULING_PREEMPTION_THRESHOLD]
0: [--schedule-conservativeness SCHEDULE_CONSERVATIVENESS]
0: [--page-size PAGE_SIZE]
0: [--hybrid-kvcache-ratio [HYBRID_KVCACHE_RATIO]]
0: [--swa-full-tokens-ratio SWA_FULL_TOKENS_RATIO]
0: [--disable-hybrid-swa-memory]
0: [--device DEVICE]
0: [--tensor-parallel-size TENSOR_PARALLEL_SIZE]
0: [--pipeline-parallel-size PIPELINE_PARALLEL_SIZE]
0: [--max-micro-batch-size MAX_MICRO_BATCH_SIZE]
0: [--stream-interval STREAM_INTERVAL]
0: [--stream-output]
0: [--random-seed RANDOM_SEED]
0: [--constrained-json-whitespace-pattern CONSTRAINED_JSON_WHITESPACE_PATTERN]
0: [--watchdog-timeout WATCHDOG_TIMEOUT]
0: [--dist-timeout DIST_TIMEOUT]
0: [--download-dir DOWNLOAD_DIR]
0: [--base-gpu-id BASE_GPU_ID]
0: [--gpu-id-step GPU_ID_STEP]
0: [--sleep-on-idle] [--log-level LOG_LEVEL]
0: [--log-level-http LOG_LEVEL_HTTP]
0: [--log-requests]
0: [--log-requests-level {0,1,2,3}]
0: [--crash-dump-folder CRASH_DUMP_FOLDER]
0: [--show-time-cost] [--enable-metrics]
0: [--enable-metrics-for-all-schedulers]
0: [--tokenizer-metrics-custom-labels-header TOKENIZER_METRICS_CUSTOM_LABELS_HEADER]
0: [--tokenizer-metrics-allowed-customer-labels TOKENIZER_METRICS_ALLOWED_CUSTOMER_LABELS [TOKENIZER_METRICS_ALLOWED_CUSTOMER_LABELS ...]]
0: [--bucket-time-to-first-token BUCKET_TIME_TO_FIRST_TOKEN [BUCKET_TIME_TO_FIRST_TOKEN ...]]
0: [--bucket-inter-token-latency BUCKET_INTER_TOKEN_LATENCY [BUCKET_INTER_TOKEN_LATENCY ...]]
0: [--bucket-e2e-request-latency BUCKET_E2E_REQUEST_LATENCY [BUCKET_E2E_REQUEST_LATENCY ...]]
0: [--collect-tokens-histogram]
0: [--prompt-tokens-buckets PROMPT_TOKENS_BUCKETS [PROMPT_TOKENS_BUCKETS ...]]
0: [--generation-tokens-buckets GENERATION_TOKENS_BUCKETS [GENERATION_TOKENS_BUCKETS ...]]
0: [--gc-warning-threshold-secs GC_WARNING_THRESHOLD_SECS]
0: [--decode-log-interval DECODE_LOG_INTERVAL]
0: [--enable-request-time-stats-logging]
0: [--kv-events-config KV_EVENTS_CONFIG]
0: [--enable-trace]
0: [--oltp-traces-endpoint OLTP_TRACES_ENDPOINT]
0: [--api-key API_KEY]
0: [--served-model-name SERVED_MODEL_NAME]
0: [--weight-version WEIGHT_VERSION]
0: [--chat-template CHAT_TEMPLATE]
0: [--completion-template COMPLETION_TEMPLATE]
0: [--file-storage-path FILE_STORAGE_PATH]
0: [--enable-cache-report]
0: [--reasoning-parser {deepseek-r1,deepseek-v3,glm45,gpt-oss,kimi,qwen3,qwen3-thinking,step3}]
0: [--tool-call-parser {llama3,qwen25,mistral,deepseekv3,deepseekv31,pythonic,kimi_k2,qwen3_coder,glm45,step3,gpt-oss}]
0: [--tool-server TOOL_SERVER]
0: [--data-parallel-size DATA_PARALLEL_SIZE]
0: [--load-balance-method {round_robin,shortest_queue,minimum_tokens}]
0: [--load-watch-interval LOAD_WATCH_INTERVAL]
0: [--prefill-round-robin-balance]
0: [--dist-init-addr DIST_INIT_ADDR]
0: [--nnodes NNODES] [--node-rank NODE_RANK]
0: [--json-model-override-args JSON_MODEL_OVERRIDE_ARGS]
0: [--preferred-sampling-params PREFERRED_SAMPLING_PARAMS]
0: [--enable-lora]
0: [--max-lora-rank MAX_LORA_RANK]
0: [--lora-target-modules [{q_proj,k_proj,v_proj,o_proj,gate_proj,up_proj,down_proj,qkv_proj,gate_up_proj,all} ...]]
0: [--lora-paths [LORA_PATHS ...]]
0: [--max-loras-per-batch MAX_LORAS_PER_BATCH]
0: [--max-loaded-loras MAX_LOADED_LORAS]
0: [--lora-backend {triton,csgmv}]
0: [--attention-backend {triton,torch_native,flex_attention,cutlass_mla,fa3,fa4,flashinfer,flashmla,trtllm_mla,trtllm_mha,dual_chunk_flash_attn,hybrid_linear_attn,aiter,wave,intel_amx,ascend}]
0: [--prefill-attention-backend {triton,torch_native,flex_attention,cutlass_mla,fa3,fa4,flashinfer,flashmla,trtllm_mla,trtllm_mha,dual_chunk_flash_attn,hybrid_linear_attn,aiter,wave,intel_amx,ascend}]
0: [--decode-attention-backend {triton,torch_native,flex_attention,cutlass_mla,fa3,fa4,flashinfer,flashmla,trtllm_mla,trtllm_mha,dual_chunk_flash_attn,hybrid_linear_attn,aiter,wave,intel_amx,ascend}]
0: [--sampling-backend {flashinfer,pytorch}]
0: [--grammar-backend {xgrammar,outlines,llguidance,none}]
0: [--mm-attention-backend {sdpa,fa3,triton_attn}]
0: [--speculative-algorithm {EAGLE,EAGLE3,NEXTN,STANDALONE,LOOKAHEAD}]
0: [--speculative-draft-model-path SPECULATIVE_DRAFT_MODEL_PATH]
0: [--speculative-draft-model-revision SPECULATIVE_DRAFT_MODEL_REVISION]
0: [--speculative-num-steps SPECULATIVE_NUM_STEPS]
0: [--speculative-eagle-topk SPECULATIVE_EAGLE_TOPK]
0: [--speculative-num-draft-tokens SPECULATIVE_NUM_DRAFT_TOKENS]
0: [--speculative-accept-threshold-single SPECULATIVE_ACCEPT_THRESHOLD_SINGLE]
0: [--speculative-accept-threshold-acc SPECULATIVE_ACCEPT_THRESHOLD_ACC]
0: [--speculative-token-map SPECULATIVE_TOKEN_MAP]
0: [--speculative-attention-mode {prefill,decode}]
0: [--speculative-lookahead-min-match-window-size SPECULATIVE_LOOKAHEAD_MIN_MATCH_WINDOW_SIZE]
0: [--speculative-lookahead-max-match-window-size SPECULATIVE_LOOKAHEAD_MAX_MATCH_WINDOW_SIZE]
0: [--speculative-lookahead-min-bfs-breadth SPECULATIVE_LOOKAHEAD_MIN_BFS_BREADTH]
0: [--speculative-lookahead-max-bfs-breadth SPECULATIVE_LOOKAHEAD_MAX_BFS_BREADTH]
0: [--speculative-lookahead-match-type {BFS,PROB}]
0: [--speculative-lookahead-branch-length SPECULATIVE_LOOKAHEAD_BRANCH_LENGTH]
0: [--speculative-lookahead-capacity SPECULATIVE_LOOKAHEAD_CAPACITY]
0: [--expert-parallel-size EXPERT_PARALLEL_SIZE]
0: [--moe-a2a-backend {none,deepep}]
0: [--moe-runner-backend {auto,triton,triton_kernel,flashinfer_trtllm,flashinfer_cutlass,flashinfer_mxfp4,flashinfer_cutedsl}]
0: [--flashinfer-mxfp4-moe-precision {default,bf16}]
0: [--enable-flashinfer-allreduce-fusion]
0: [--deepep-mode {normal,low_latency,auto}]
0: [--ep-num-redundant-experts EP_NUM_REDUNDANT_EXPERTS]
0: [--ep-dispatch-algorithm EP_DISPATCH_ALGORITHM]
0: [--init-expert-location INIT_EXPERT_LOCATION]
0: [--enable-eplb]
0: [--eplb-algorithm EPLB_ALGORITHM]
0: [--eplb-rebalance-num-iterations EPLB_REBALANCE_NUM_ITERATIONS]
0: [--eplb-rebalance-layers-per-chunk EPLB_REBALANCE_LAYERS_PER_CHUNK]
0: [--eplb-min-rebalancing-utilization-threshold EPLB_MIN_REBALANCING_UTILIZATION_THRESHOLD]
0: [--expert-distribution-recorder-mode EXPERT_DISTRIBUTION_RECORDER_MODE]
0: [--expert-distribution-recorder-buffer-size EXPERT_DISTRIBUTION_RECORDER_BUFFER_SIZE]
0: [--enable-expert-distribution-metrics]
0: [--deepep-config DEEPEP_CONFIG]
0: [--moe-dense-tp-size MOE_DENSE_TP_SIZE]
0: [--max-mamba-cache-size MAX_MAMBA_CACHE_SIZE]
0: [--mamba-ssm-dtype {float32,bfloat16}]
0: [--enable-hierarchical-cache]
0: [--hicache-ratio HICACHE_RATIO]
0: [--hicache-size HICACHE_SIZE]
0: [--hicache-write-policy {write_back,write_through,write_through_selective}]
0: [--radix-eviction-policy {lru,lfu}]
0: [--hicache-io-backend {direct,kernel}]
0: [--hicache-mem-layout {layer_first,page_first,page_first_direct}]
0: [--hicache-storage-backend {file,mooncake,hf3fs,nixl}]
0: [--hicache-storage-prefetch-policy {best_effort,wait_complete,timeout}]
0: [--hicache-storage-backend-extra-config HICACHE_STORAGE_BACKEND_EXTRA_CONFIG]
0: [--enable-lmcache]
0: [--enable-double-sparsity]
0: [--ds-channel-config-path DS_CHANNEL_CONFIG_PATH]
0: [--ds-heavy-channel-num DS_HEAVY_CHANNEL_NUM]
0: [--ds-heavy-token-num DS_HEAVY_TOKEN_NUM]
0: [--ds-heavy-channel-type DS_HEAVY_CHANNEL_TYPE]
0: [--ds-sparse-decode-threshold DS_SPARSE_DECODE_THRESHOLD]
0: [--cpu-offload-gb CPU_OFFLOAD_GB]
0: [--offload-group-size OFFLOAD_GROUP_SIZE]
0: [--offload-num-in-group OFFLOAD_NUM_IN_GROUP]
0: [--offload-prefetch-step OFFLOAD_PREFETCH_STEP]
0: [--offload-mode OFFLOAD_MODE]
0: [--disable-radix-cache]
0: [--cuda-graph-max-bs CUDA_GRAPH_MAX_BS]
0: [--cuda-graph-bs CUDA_GRAPH_BS [CUDA_GRAPH_BS ...]]
0: [--disable-cuda-graph]
0: [--disable-cuda-graph-padding]
0: [--enable-profile-cuda-graph]
0: [--enable-cudagraph-gc]
0: [--enable-nccl-nvls] [--enable-symm-mem]
0: [--disable-flashinfer-cutlass-moe-fp4-allgather]
0: [--enable-tokenizer-batch-encode]
0: [--disable-outlines-disk-cache]
0: [--disable-custom-all-reduce]
0: [--enable-mscclpp]
0: [--disable-overlap-schedule]
0: [--enable-mixed-chunk]
0: [--enable-dp-attention]
0: [--enable-dp-lm-head]
0: [--enable-two-batch-overlap]
0: [--tbo-token-distribution-threshold TBO_TOKEN_DISTRIBUTION_THRESHOLD]
0: [--enable-torch-compile]
0: [--torch-compile-max-bs TORCH_COMPILE_MAX_BS]
0: [--torchao-config TORCHAO_CONFIG]
0: [--enable-nan-detection]
0: [--enable-p2p-check]
0: [--triton-attention-reduce-in-fp32]
0: [--triton-attention-num-kv-splits TRITON_ATTENTION_NUM_KV_SPLITS]
0: [--triton-attention-split-tile-size TRITON_ATTENTION_SPLIT_TILE_SIZE]
0: [--num-continuous-decode-steps NUM_CONTINUOUS_DECODE_STEPS]
0: [--delete-ckpt-after-loading]
0: [--enable-memory-saver]
0: [--allow-auto-truncate]
0: [--enable-custom-logit-processor]
0: [--flashinfer-mla-disable-ragged]
0: [--disable-shared-experts-fusion]
0: [--disable-chunked-prefix-cache]
0: [--disable-fast-image-processor]
0: [--keep-mm-feature-on-device]
0: [--enable-return-hidden-states]
0: [--scheduler-recv-interval SCHEDULER_RECV_INTERVAL]
0: [--numa-node NUMA_NODE [NUMA_NODE ...]]
0: [--debug-tensor-dump-output-folder DEBUG_TENSOR_DUMP_OUTPUT_FOLDER]
0: [--debug-tensor-dump-input-file DEBUG_TENSOR_DUMP_INPUT_FILE]
0: [--debug-tensor-dump-inject DEBUG_TENSOR_DUMP_INJECT]
0: [--debug-tensor-dump-prefill-only]
0: [--enable-dynamic-batch-tokenizer]
0: [--dynamic-batch-tokenizer-batch-size DYNAMIC_BATCH_TOKENIZER_BATCH_SIZE]
0: [--dynamic-batch-tokenizer-batch-timeout DYNAMIC_BATCH_TOKENIZER_BATCH_TIMEOUT]
0: [--disaggregation-mode {null,prefill,decode}]
0: [--disaggregation-transfer-backend {mooncake,nixl,ascend,fake}]
0: [--disaggregation-bootstrap-port DISAGGREGATION_BOOTSTRAP_PORT]
0: [--disaggregation-decode-tp DISAGGREGATION_DECODE_TP]
0: [--disaggregation-decode-dp DISAGGREGATION_DECODE_DP]
0: [--disaggregation-prefill-pp DISAGGREGATION_PREFILL_PP]
0: [--disaggregation-ib-device DISAGGREGATION_IB_DEVICE]
0: [--num-reserved-decode-tokens NUM_RESERVED_DECODE_TOKENS]
0: [--disaggregation-decode-polling-interval DISAGGREGATION_DECODE_POLLING_INTERVAL]
0: [--custom-weight-loader [CUSTOM_WEIGHT_LOADER ...]]
0: [--weight-loader-disable-mmap]
0: [--enable-pdmux]
0: [--sm-group-num SM_GROUP_NUM]
0: [--enable-ep-moe] [--enable-deepep-moe]
0: [--enable-flashinfer-cutlass-moe]
0: [--enable-flashinfer-cutedsl-moe]
0: [--enable-flashinfer-trtllm-moe]
0: [--enable-triton-kernel-moe]
0: [--enable-flashinfer-mxfp4-moe]
0: [--backend BACKEND]
0: [--result-filename RESULT_FILENAME]
0: [--dataset-name {sharegpt,random,generated-shared-prefix}]
0: [--dataset-path DATASET_PATH]
0: [--num-prompts NUM_PROMPTS]
0: [--sharegpt-output-len SHAREGPT_OUTPUT_LEN]
0: [--sharegpt-context-len SHAREGPT_CONTEXT_LEN]
0: [--random-input-len RANDOM_INPUT_LEN]
0: [--random-output-len RANDOM_OUTPUT_LEN]
0: [--random-range-ratio RANDOM_RANGE_RATIO]
0: [--gsp-num-groups GSP_NUM_GROUPS]
0: [--gsp-prompts-per-group GSP_PROMPTS_PER_GROUP]
0: [--gsp-system-prompt-len GSP_SYSTEM_PROMPT_LEN]
0: [--gsp-question-len GSP_QUESTION_LEN]
0: [--gsp-output-len GSP_OUTPUT_LEN]
0: [--seed SEED] [--disable-ignore-eos]
0: [--extra-request-body {"key1": "value1", "key2": "value2"}]
0: [--apply-chat-template] [--profile]
0: [--skip-warmup] [--do-not-exit]
0: [--prompt-suffix PROMPT_SUFFIX]
sglang.bench_offline_throughput
CLI reference for measuring SGLang offline throughput.