diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
index 15a9a022..dafd20bf 100644
--- a/.github/workflows/rust.yml
+++ b/.github/workflows/rust.yml
@@ -43,7 +43,7 @@ jobs:
 
     strategy:
       matrix:
-        rust: [nightly]
+        rust: [nightly-2022-03-24]
         features: ["", "rayon", "nightly_aarch64_neon"]
 
     steps:
diff --git a/src/arch/neon.rs b/src/arch/neon.rs
index 36450c07..48435789 100644
--- a/src/arch/neon.rs
+++ b/src/arch/neon.rs
@@ -3,7 +3,6 @@ use core::arch::aarch64::*;
 
 #[cfg(all(feature = "nightly_aarch64_neon", target_arch = "aarch64"))]
 #[target_feature(enable = "neon")]
-#[target_feature(enable = "fp")]
 unsafe fn idct8(data: &mut [int16x8_t; 8]) {
     // The fixed-point constants here are obtained by taking the fractional part of the constants
     // from the non-SIMD implementation and scaling them up by 1<<15. This is because
@@ -84,7 +83,6 @@ unsafe fn idct8(data: &mut [int16x8_t; 8]) {
 
 #[cfg(all(feature = "nightly_aarch64_neon", target_arch = "aarch64"))]
 #[target_feature(enable = "neon")]
-#[target_feature(enable = "fp")]
 unsafe fn transpose8(data: &mut [int16x8_t; 8]) {
     // Use NEON's 2x2 matrix transposes (vtrn) to do the transposition in each 4x4 block, then
     // combine the 4x4 blocks.
@@ -112,7 +110,6 @@ unsafe fn transpose8(data: &mut [int16x8_t; 8]) {
 
 #[cfg(all(feature = "nightly_aarch64_neon", target_arch = "aarch64"))]
 #[target_feature(enable = "neon")]
-#[target_feature(enable = "fp")]
 pub unsafe fn dequantize_and_idct_block_8x8(
     coefficients: &[i16; 64],
     quantization_table: &[u16; 64],
@@ -171,7 +168,6 @@ pub unsafe fn dequantize_and_idct_block_8x8(
 
 #[cfg(all(feature = "nightly_aarch64_neon", target_arch = "aarch64"))]
 #[target_feature(enable = "neon")]
-#[target_feature(enable = "fp")]
 pub unsafe fn color_convert_line_ycbcr(y: &[u8], cb: &[u8], cr: &[u8], output: &mut [u8]) -> usize {
     assert!(output.len() % 3 == 0);
     let num = output.len() / 3;