mixed_precision: fp16 # or bf16 mixed_precision: bf16 # or fp16 num_machines: 1 # the number of nodes num_processes: 2 # the number of GPUs in all nodes rdzv_backend: static 2 changes: 1 addition & 1 deletion 2 examples/deepspeed/ds_z0_config.json Original file line numberDiff...
File metadata and controls Code Blame 16 lines (16 loc) · 378 Bytes Raw { "train_micro_batch_size_per_gpu": 8, "gradient_accumulation_steps": 1, "zero_optimization": { "stage": 1 }, "zero_allow_untested_optimizer": true, "fp16": { "enabled": true, "loss_scale": 0, "init...
{ "train_batch_size": "auto", "train_micro_batch_size_per_gpu": "auto", "gradient_accumulation_steps": "auto", "gradient_clipping": "auto", "zero_allow_untested_optimizer": true, "fp16": { "enabled": "auto", "loss_scale": 0, "loss_scale_window": 1000, "initial_...
{ "train_micro_batch_size_per_gpu": 8, "gradient_accumulation_steps": 1, "zero_optimization": { "stage": 1 }, "zero_allow_untested_optimizer": true, "fp16": { "enabled": true, "loss_scale": 0, "initial_scale_power": 11, "loss_scale_window": 1000, "hysteresis": 16 ...
mixed_precision:bf16#orfp16 1818 num_machines:1#the number of nodes 1919 num_processes:2#the number of GPUs in all nodes 2020 rdzv_backend:static examples/deepspeed/ds_z0_config.json +1-1 Original file line numberDiff line numberDiff line change ...
Security1 Insights Additional navigation options Files b29d556 .github assets data evaluation examples accelerate deepspeed ds_z2_config.json ds_z2_offload_config.json ds_z3_config.json ds_z3_offload_config.json extras fsdp_qlora full_multi_gpu ...