diff --git a/packaging/pkg_helpers.bash b/packaging/pkg_helpers.bash
index 28f1c9acf4b..29a2ea096e4 100644
--- a/packaging/pkg_helpers.bash
+++ b/packaging/pkg_helpers.bash
@@ -246,9 +246,9 @@ setup_conda_pytorch_constraint() {
 
 # Translate CUDA_VERSION into CUDA_CUDATOOLKIT_CONSTRAINT
 setup_conda_cudatoolkit_constraint() {
-  export CONDA_CPUONLY_FEATURE=""
+  export CONDA_BUILD_VARIANT="cuda"
   if [[ "$(uname)" == Darwin ]]; then
-    export CONDA_CUDATOOLKIT_CONSTRAINT=""
+    export CONDA_BUILD_VARIANT="cpu"
   else
     case "$CU_VERSION" in
       cu113)
@@ -277,7 +277,7 @@ setup_conda_cudatoolkit_constraint() {
         ;;
       cpu)
         export CONDA_CUDATOOLKIT_CONSTRAINT=""
-        export CONDA_CPUONLY_FEATURE="- cpuonly"
+        export CONDA_BUILD_VARIANT="cpu"
         ;;
       *)
         echo "Unrecognized CU_VERSION=$CU_VERSION"
diff --git a/packaging/torchaudio/meta.yaml b/packaging/torchaudio/meta.yaml
index 720c396d930..64ea46e9375 100644
--- a/packaging/torchaudio/meta.yaml
+++ b/packaging/torchaudio/meta.yaml
@@ -1,3 +1,4 @@
+{% set build_variant = environ.get('CONDA_BUILD_VARIANT', 'cpu') %}
 package:
   name: torchaudio
   version: "{{ environ.get('BUILD_VERSION', '0.0.0') }}"
@@ -18,23 +19,29 @@ requirements:
     - defaults::numpy >=1.11
     {{ environ.get('CONDA_PYTORCH_BUILD_CONSTRAINT', 'pytorch') }}
     {{ environ.get('CONDA_EXTRA_BUILD_CONSTRAINT', '') }}
-    {{ environ.get('CONDA_CPUONLY_FEATURE', '') }}
     {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
 
   run:
     - python
     - defaults::numpy >=1.11
+    - pytorch-mutex 1.0 {{ build_variant }}  # [not osx ]
     {{ environ.get('CONDA_PYTORCH_CONSTRAINT', 'pytorch') }}
     {{ environ.get('CONDA_CUDATOOLKIT_CONSTRAINT', '') }}
 
+  {% if build_variant == 'cpu' %}
+  run_constrained:
+    - cpuonly
+  {% elif not osx %}
+  run_constrained:
+     - cpuonly <0
+  {% endif %}
+
 build:
   string: py{{py}}_{{ environ.get('CU_VERSION', 'cpu') }}
   script_env:
     - BUILD_VERSION
     - USE_CUDA
     - TORCH_CUDA_ARCH_LIST
-  features:
-    {{ environ.get('CONDA_CPUONLY_FEATURE', '') }}
 
 test:
   imports:
@@ -52,7 +59,7 @@ test:
     # Ideally we would test this, but conda doesn't provide librosa
     # - librosa >=0.4.3
     - scipy
-    {{ environ.get('CONDA_CPUONLY_FEATURE', '') }}
+    - pytorch-mutex 1.0 {{ build_variant }}  # [not osx ]
 
 about:
   home: https://github.com/pytorch/audio