diff --git a/.github/workflows/package-checks.yml b/.github/workflows/package-checks.yml
index 762cb509be136..87f40270d8774 100644
--- a/.github/workflows/package-checks.yml
+++ b/.github/workflows/package-checks.yml
@@ -9,12 +9,14 @@ on:
     branches:
       - main
       - 1.5.x
+    types: [ labeled, opened, synchronize, reopened ]
 
 permissions:
   contents: read
 
 jobs:
   pip:
+    if: ${{ github.event.label.name == 'Build' || contains(github.event.pull_request.labels.*.name, 'Build') || github.event_name == 'push'}}
     runs-on: ubuntu-latest
     strategy:
       matrix:
@@ -38,13 +40,6 @@ jobs:
         with:
           python-version: '3.8'
 
-      # Hacky patch to disable building cython extensions.
-      # This job should only check that the extras successfully install.
-      - name: Disable building ext_modules
-        run: |
-          sed -i '/ext_modules=/d' setup.py
-        shell: bash -el {0}
-
       - name: Install required dependencies
         run: |
           python -m pip install --upgrade pip setuptools wheel python-dateutil pytz numpy cython