diff --git a/configure.ac b/configure.ac
index 5eccef2fde6323460cc0d55c109a054767a33313..d6cf46ad95ea620ef132c8d4bd10d463fe660898 100644
--- a/configure.ac
+++ b/configure.ac
@@ -178,16 +178,6 @@ if test "$enable_opt" = "yes" ; then
    AX_CC_MAXOPT
    ac_test_CFLAGS="yes"
    CFLAGS="$old_CFLAGS $CFLAGS"
-
-   # Check SSE & AVX support (some overlap with AX_CC_MAXOPT).
-   # Don't use the SIMD_FLAGS result with Intel compilers. The -x<code>
-   # value from AX_CC_MAXOPT should be sufficient.
-   AX_EXT
-   if test "$SIMD_FLAGS" != ""; then
-       if test "$ax_cv_c_compiler_vendor" != "intel"; then
-           CFLAGS="$CFLAGS $SIMD_FLAGS"
-       fi
-   fi
 fi
 
 # Add address sanitizer options to flags, if requested. Only useful for GCC
@@ -242,6 +232,17 @@ if test "$enable_vec" = "no"; then
    fi
    HAVEVECTORIZATION=0
 else
+
+   # Check SSE & AVX support (some overlap with AX_CC_MAXOPT).
+   # Don't use the SIMD_FLAGS result with Intel compilers. The -x<code>
+   # value from AX_CC_MAXOPT should be sufficient.
+   AX_EXT
+   if test "$SIMD_FLAGS" != ""; then
+       if test "$ax_cv_c_compiler_vendor" != "intel"; then
+           CFLAGS="$CFLAGS $SIMD_FLAGS"
+       fi
+   fi
+
    AC_DEFINE([WITH_VECTORIZATION],1,[Enable vectorization])
    HAVEVECTORIZATION=1
 fi