diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -56,8 +56,6 @@
 
   # Allow extra cmake option to be specified for each host
   declare -A CMAKE_EXTRA_OPTIONS
-  CMAKE_EXTRA_OPTIONS[i686-pc-linux-gnu]="-DSECP256K1_USE_ASM=OFF"
-  CMAKE_EXTRA_OPTIONS[aarch64-linux-gnu]="-DSECP256K1_USE_ASM=OFF"
   # ARM assembly is supported but experimental, disable it for the release
   CMAKE_EXTRA_OPTIONS[arm-linux-gnueabihf]="-DSECP256K1_USE_ASM=OFF"
 
diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml
--- a/contrib/gitian-descriptors/gitian-win.yml
+++ b/contrib/gitian-descriptors/gitian-win.yml
@@ -164,7 +164,7 @@
 
   # Allow extra cmake option to be specified for each host
   declare -A CMAKE_EXTRA_OPTIONS
-  CMAKE_EXTRA_OPTIONS[i686-w64-mingw32]="-DCPACK_PACKAGE_FILE_NAME=${DISTNAME}-win32-setup-unsigned -DSECP256K1_USE_ASM=OFF"
+  CMAKE_EXTRA_OPTIONS[i686-w64-mingw32]="-DCPACK_PACKAGE_FILE_NAME=${DISTNAME}-win32-setup-unsigned"
   CMAKE_EXTRA_OPTIONS[x86_64-w64-mingw32]="-DCPACK_PACKAGE_FILE_NAME=${DISTNAME}-win64-setup-unsigned"
 
   ORIGPATH="$PATH"
diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt
--- a/src/secp256k1/CMakeLists.txt
+++ b/src/secp256k1/CMakeLists.txt
@@ -65,21 +65,31 @@
 	set(USE_SCALAR_INV_BUILTIN 1)
 endif()
 
-option(SECP256K1_USE_ASM "Use assembly if available" ON)
+# Guess the target architecture, within the ones with supported ASM.
+# First check if the CMAKE_C_COMPILER_TARGET is set (should be when
+# cross compiling), then CMAKE_SYSTEM_PROCESSOR as a fallback if meaningful
+# (this is not the case for ARM as the content is highly non standard).
+if(CMAKE_C_COMPILER_TARGET MATCHES "x86_64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
+	set(SECP256K1_ASM_BUILD_TARGET "x86_64")
+	set(SECP256K1_DEFAULT_USE_ASM ON)
+elseif(CMAKE_C_COMPILER_TARGET MATCHES "arm-linux-gnueabihf")
+	set(SECP256K1_ASM_BUILD_TARGET "arm-linux-gnueabihf")
+	set(SECP256K1_DEFAULT_USE_ASM ON)
+endif()
+
+# Enable ASM by default only if we are building for a compatible target.
+# The user can still enable/disable it manually if needed.
+option(SECP256K1_USE_ASM "Use assembly if available" ${SECP256K1_DEFAULT_USE_ASM})
 
 if(SECP256K1_USE_ASM)
 	macro(unsupported_asm_error)
 		message(FATAL_ERROR
-			"Assembly is not supported for your target architecture."
-			"Re-run cmake with -DSECP256K1_USE_ASM=OFF to disable ASM support"
+			"Assembly is enabled, but not supported for your target architecture."
+			"Re-run cmake with -DSECP256K1_USE_ASM=OFF to disable ASM support."
 		)
 	endmacro()
 
-	# Guess the target architecture.
-	# First check if the CMAKE_C_COMPILER_TARGET is set (should be when
-	# cross compiling), then CMAKE_SYSTEM_PROCESSOR as a fallback if meaningful
-	# (this is not the case for ARM as the content is highly non standard).
-	if(CMAKE_C_COMPILER_TARGET MATCHES "x86_64" OR CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64")
+	if(SECP256K1_ASM_BUILD_TARGET MATCHES "x86_64")
 		# We check if amd64 asm is supported.
 		check_c_source_compiles("
 			#include <stdint.h>
@@ -93,7 +103,7 @@
 		if(NOT USE_ASM_X86_64)
 			unsupported_asm_error()
 		endif()
-	elseif(CMAKE_C_COMPILER_TARGET MATCHES "arm-linux-gnueabihf")
+	elseif(SECP256K1_ASM_BUILD_TARGET MATCHES "arm-linux-gnueabihf")
 		enable_language(ASM)
 		set(USE_EXTERNAL_ASM 1)
 		add_library(secp256k1_common src/asm/field_10x26_arm.s)