diff --git a/setup.py b/setup.py
index f080503fe..b5b75b73c 100644
--- a/setup.py
+++ b/setup.py
@@ -404,6 +404,8 @@ class BuildExtension(build_ext.build_ext):
             arm64_name,
         ]
         _invoke_shell_command(lipo_command)
+        # Delete the arm64 file (the x86 file was overwritten by lipo)
+        _invoke_shell_command(['rm', arm64_name])
     else:
       for ext in self.extensions:
         self._build_binary(ext)