skipping winograd in single gpu test (#2751)

* skipping winograd in single gpu test * Update old test rocm scripts to use new bazelrc files if available * merge conflict artifact fix --------- Co-authored-by: Jason Furmanek <[email protected]>
ROCm · Nov 19, 2024 · 3a49cc5 · 3a49cc5
1 parent fdff8af
commit 3a49cc5
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 44 deletions.
diff --git a/tensorflow/tools/ci_build/linux/rocm/run_cpu.sh b/tensorflow/tools/ci_build/linux/rocm/run_cpu.sh
@@ -30,19 +30,38 @@ export TF_PYTHON_VERSION=$PYTHON_VERSION
 
 export TF_NEED_ROCM=0
 
-yes "" | $PYTHON_BIN_PATH configure.py
+if [ -f /usertools/cpu.bazelrc ]; then
+        # Use the bazelrc files in /usertools if available
+        bazel \
+          --bazelrc=/usertools/cpu.bazelrc \
+          test \
+          --config=sigbuild_local_cache \
+          --config=pycpp \
+          --action_env=TF_PYTHON_VERSION=$PYTHON_VERSION \
+          --local_test_jobs=${N_BUILD_JOBS} \
+          --jobs=${N_BUILD_JOBS}
+else
+         yes "" | $PYTHON_BIN_PATH configure.py
 
-bazel test \
-      -k \
-      --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-multi_gpu,-tpu,-cuda-only,-benchmark-test,-v1only \
-      --jobs=${N_BUILD_JOBS} \
-      --local_test_jobs=${N_BUILD_JOBS} \
-      --test_timeout 600,900,2400,7200 \
-      --build_tests_only \
-      --test_output=errors \
-      --test_sharding_strategy=disabled \
-      --test_size_filters=small,medium,large \
-      -- \
-      //tensorflow/... \
-      -//tensorflow/core/tpu/... \
-      -//tensorflow/lite/... \
+        # Run bazel test command. Double test timeouts to avoid flakes.
+        # xla/mlir_hlo/tests/Dialect/gml_st tests disabled in 09/08/22 sync
+        bazel test \
+              -k \
+              --test_tag_filters=-no_oss,-oss_excluded,-oss_serial,-gpu,-multi_gpu,-tpu,-no_rocm,-benchmark-test,-v1only \
+              --test_lang_filters=cc,py \
+              --jobs=${N_BUILD_JOBS} \
+              --local_test_jobs=${N_BUILD_JOBS} \
+              --test_timeout 920,2400,7200,9600 \
+              --build_tests_only \
+              --test_output=errors \
+              --test_sharding_strategy=disabled \
+              --test_size_filters=small,medium \
+              --test_env=TF_PYTHON_VERSION=$PYTHON_VERSION \
+              -- \
+              //tensorflow/... \
+              -//tensorflow/python/integration_testing/... \
+              -//tensorflow/compiler/tf2tensorrt/... \
+              -//tensorflow/core/tpu/... \
+              -//tensorflow/lite/... \
+              -//tensorflow/tools/toolchains/...
+fi
diff --git a/tensorflow/tools/ci_build/linux/rocm/run_gpu_multi.sh b/tensorflow/tools/ci_build/linux/rocm/run_gpu_multi.sh
@@ -29,7 +29,7 @@ if [[ -n $1 ]]; then
     ROCM_INSTALL_DIR=$1
 else
     if [[ -z "${ROCM_PATH}" ]]; then
-        ROCM_INSTALL_DIR=/opt/rocm-6.2.0
+        ROCM_INSTALL_DIR=/opt/rocm-6.0.0
     else
         ROCM_INSTALL_DIR=$ROCM_PATH
     fi
@@ -45,17 +45,11 @@ export ROCM_PATH=$ROCM_INSTALL_DIR
 
 if [ -f /usertools/rocm.bazelrc ]; then
 	 # Use the bazelrc files in /usertools if available
-  	if [ ! -d /tf ];then
-	   # The bazelrc files in /usertools expect /tf to exist
-           mkdir /tf
-	fi
 	bazel \
            --bazelrc=/usertools/rocm.bazelrc \
            test \
            --local_test_jobs=${N_TEST_JOBS} \
-           --jobs=30 \
-           --local_ram_resources=60000 \
-           --local_cpu_resources=15 \
+           --jobs=${N_BUILD_JOBS} \
            --config=sigbuild_local_cache \
            --config=rocm \
            --config=nonpip_multi_gpu \

diff --git a/tensorflow/tools/ci_build/linux/rocm/run_gpu_single.sh b/tensorflow/tools/ci_build/linux/rocm/run_gpu_single.sh
@@ -51,25 +51,46 @@ export TF_PYTHON_VERSION=$PYTHON_VERSION
 export TF_NEED_ROCM=1
 export ROCM_PATH=$ROCM_INSTALL_DIR
 
-yes "" | $PYTHON_BIN_PATH configure.py
+if [ -f /usertools/rocm.bazelrc ]; then
+	# Use the bazelrc files in /usertools if available
+	bazel \
+	     --bazelrc=/usertools/rocm.bazelrc \
+             test \
+             --jobs=${N_BUILD_JOBS} \
+	     --local_test_jobs=${N_TEST_JOBS} \
+             --config=sigbuild_local_cache \
+             --config=rocm \
+             --config=pycpp \
+             --action_env=TF_PYTHON_VERSION=$PYTHON_VERSION \
+             --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+             --test_env=TF_GPU_COUNT=$TF_GPU_COUNT
+else
+	# Legacy style: run configure then build
+	yes "" | $PYTHON_BIN_PATH configure.py
 
-# Run bazel test command. Double test timeouts to avoid flakes.
-bazel test \
-      --config=rocm \
-      -k \
-      --test_tag_filters=gpu,-no_oss,-oss_excluded,-oss_serial,-no_gpu,-cuda-only,-benchmark-test,-rocm_multi_gpu,-tpu,-v1only \
-      --jobs=${N_BUILD_JOBS} \
-      --local_test_jobs=${N_TEST_JOBS} \
-      --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
-      --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
-      --test_timeout 600,900,2400,7200 \
-      --build_tests_only \
-      --test_output=errors \
-      --test_sharding_strategy=disabled \
-      --test_size_filters=small,medium,large \
-      --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \
-      -- \
-      //tensorflow/... \
-      -//tensorflow/core/tpu/... \
-      -//tensorflow/lite/... \
-      -//tensorflow/compiler/tf2tensorrt/... \
+  # Run bazel test command. Double test timeouts to avoid flakes.
+	bazel test \
+	      --config=rocm \
+	      -k \
+	      --test_tag_filters=gpu,-no_oss,-oss_excluded,-oss_serial,-no_gpu,-no_rocm,-benchmark-test,-rocm_multi_gpu,-tpu,-v1only \
+	      --jobs=${N_BUILD_JOBS} \
+	      --local_test_jobs=${N_TEST_JOBS} \
+	      --test_env=TF_GPU_COUNT=$TF_GPU_COUNT \
+	      --test_env=TF_TESTS_PER_GPU=$TF_TESTS_PER_GPU \
+	      --test_env=HSA_TOOLS_LIB=libroctracer64.so \
+        --test_env=MIOPEN_DEBUG_CONV_WINOGRAD=0 \
+	      --test_env=TF_PYTHON_VERSION=$PYTHON_VERSION \
+	      --test_timeout 920,2400,7200,9600 \
+	      --build_tests_only \
+	      --test_output=errors \
+	      --test_sharding_strategy=disabled \
+	      --test_size_filters=small,medium,large \
+	      --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \
+	      -- \
+	      //tensorflow/... \
+	      -//tensorflow/python/integration_testing/... \
+	      -//tensorflow/core/tpu/... \
+	      -//tensorflow/lite/... \
+	      -//tensorflow/compiler/tf2tensorrt/... \
+	      -//tensorflow/dtensor/python/tests:multi_client_test_nccl_2gpus
+fi