diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh
index 677578ce3a4..f2286cd40b9 100644
--- a/.ci/scripts/utils.sh
+++ b/.ci/scripts/utils.sh
@@ -32,7 +32,7 @@ install_executorch() {
   which pip
   # Install executorch, this assumes that Executorch is checked out in the
   # current directory.
-  ./install_executorch.sh --pybind xnnpack "$@"
+  ./install_executorch.sh "$@"
   # Just print out the list of packages for debugging
   pip list
 }
diff --git a/install_executorch.py b/install_executorch.py
index 6863ed2c0fc..1d3fe8af1fb 100644
--- a/install_executorch.py
+++ b/install_executorch.py
@@ -207,14 +207,10 @@ def main(args):
     use_pytorch_nightly = True
 
     wants_pybindings_off, pybind_defines = _list_pybind_defines(args)
-    if not wants_pybindings_off:
-        if len(pybind_defines) > 0:
-            # If the user explicitly provides a list of bindings, just use them
-            cmake_args += pybind_defines
-        else:
-            # If the user has not set pybindings off but also has not provided
-            # a list, then turn on xnnpack by default
-            cmake_args.append("-DEXECUTORCH_BUILD_XNNPACK=ON")
+    if wants_pybindings_off:
+        cmake_args.append("-DEXECUTORCH_BUILD_PYBIND=OFF")
+    else:
+        cmake_args += pybind_defines
 
     if args.clean:
         clean()
diff --git a/setup.py b/setup.py
index 76fbbbd9025..eac28e8e26c 100644
--- a/setup.py
+++ b/setup.py
@@ -142,7 +142,7 @@ def openvino(cls) -> bool:
 
     @classmethod
     def xnnpack(cls) -> bool:
-        return cls._is_cmake_arg_enabled("EXECUTORCH_BUILD_XNNPACK", default=False)
+        return cls._is_cmake_arg_enabled("EXECUTORCH_BUILD_XNNPACK", default=True)
 
     @classmethod
     def training(cls) -> bool:
@@ -730,6 +730,9 @@ def run(self):
                 "-DEXECUTORCH_BUILD_KERNELS_QUANTIZED_AOT=ON",
             ]
 
+            if ShouldBuild.xnnpack():
+                cmake_args += ["-DEXECUTORCH_BUILD_XNNPACK=ON"]
+
             if ShouldBuild.training():
                 build_args += ["--target", "_training_lib"]