From 585a0bf60ed43fe20d1558107ccbea01c8ece65d Mon Sep 17 00:00:00 2001 From: Jimmy Zhang Date: Thu, 3 Oct 2024 08:38:27 -0700 Subject: [PATCH] allow cudnn fa Signed-off-by: Jimmy Zhang --- examples/llm/pretrain/default_executor.py | 2 -- scripts/llm/llama3_pretraining.py | 1 - 2 files changed, 3 deletions(-) diff --git a/examples/llm/pretrain/default_executor.py b/examples/llm/pretrain/default_executor.py index 2f5fc08d9d63..6ebc874b2e39 100644 --- a/examples/llm/pretrain/default_executor.py +++ b/examples/llm/pretrain/default_executor.py @@ -26,7 +26,6 @@ def local_executor_torchrun(devices: int = 2) -> run.LocalExecutor: "NCCL_NVLS_ENABLE": "0", "NVTE_DP_AMAX_REDUCE_INTERVAL": "0", "NVTE_ASYNC_AMAX_REDUCTION": "1", - "NVTE_FUSED_ATTN": "0", } executor = run.LocalExecutor(ntasks_per_node=devices, launcher="torchrun", env_vars=env_vars) @@ -63,7 +62,6 @@ def slurm_executor( "NCCL_NVLS_ENABLE": "0", "NVTE_DP_AMAX_REDUCE_INTERVAL": "0", "NVTE_ASYNC_AMAX_REDUCTION": "1", - "NVTE_FUSED_ATTN": "0", } if custom_env_vars: env_vars |= custom_env_vars diff --git a/scripts/llm/llama3_pretraining.py b/scripts/llm/llama3_pretraining.py index 2e49402a923a..a5ca3a496a88 100644 --- a/scripts/llm/llama3_pretraining.py +++ b/scripts/llm/llama3_pretraining.py @@ -84,7 +84,6 @@ def slurm_executor( "NCCL_NVLS_ENABLE": "0", "NVTE_DP_AMAX_REDUCE_INTERVAL": "0", "NVTE_ASYNC_AMAX_REDUCTION": "1", - "NVTE_FUSED_ATTN": "0", } if custom_env_vars: env_vars |= custom_env_vars