diff --git a/scripts/build_mlc_chat_wheel_manylinux.sh b/scripts/build_mlc_chat_wheel_manylinux.sh
index 3c87bad..418941e 100755
--- a/scripts/build_mlc_chat_wheel_manylinux.sh
+++ b/scripts/build_mlc_chat_wheel_manylinux.sh
@@ -79,8 +79,9 @@ else
 fi
 
 AUDITWHEEL_OPTS="--plat ${AUDITWHEEL_PLAT} -w repaired_wheels/"
+AUDITWHEEL_OPTS="--exclude libtvm --exclude libtvm_runtime --exclude libvulkan ${AUDITWHEEL_OPTS}"
 if [[ ${CUDA} != "none" ]]; then
-    AUDITWHEEL_OPTS="--exclude libcuda --exclude libcudart --exclude libnvrtc --exclude libtvm --exclude libtvm_runtime --exclude libvulkan ${AUDITWHEEL_OPTS}"
+    AUDITWHEEL_OPTS="--exclude libcuda --exclude libcudart --exclude libnvrtc ${AUDITWHEEL_OPTS}"
 fi
 
 # config the cmake