diff --git a/.github/mergify.yml b/.github/mergify.yml
index 2a1e039c..3097b994 100644
--- a/.github/mergify.yml
+++ b/.github/mergify.yml
@@ -90,9 +90,10 @@ pull_request_rules:
 
 - name: label-tpu
   description: Automatically apply tpu label
+  # Keep this list in sync with `label-tpu-remove` conditions
   conditions:
     - or:
-      - files~=tpu.
+      - files~=tpu.py
       - files~=_tpu
       - files~=tpu_
       - files~=/tpu/
@@ -102,6 +103,21 @@ pull_request_rules:
       add:
         - tpu
 
+- name: label-tpu-remove
+  description: Automatically remove tpu label
+  # Keep this list in sync with `label-tpu` conditions
+  conditions:
+    - and:
+      - -files~=tpu.py
+      - -files~=_tpu
+      - -files~=tpu_
+      - -files~=/tpu/
+      - -files~=pallas
+  actions:
+    label:
+      remove:
+        - tpu
+
 - name: ping author on conflicts and add 'needs-rebase' label
   conditions:
       - conflict