Run schedule CI against tokenizers main

huggingface · Sep 27, 2022 · e35e6c4 · e35e6c4
1 parent 157ac58
commit e35e6c4
Show file tree

Hide file tree

Showing 3 changed files with 41 additions and 15 deletions.
diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml
@@ -7,6 +7,9 @@ name: Self-hosted runner (scheduled)
 # `docker/transformers-pytorch-deepspeed-latest-gpu/Dockerfile`
 
 on:
+  push:
+    branches:
+      - post-action-build-test-tokenizers-*
   repository_dispatch:
   schedule:
     - cron: "0 2 * * *"
@@ -42,8 +45,11 @@ jobs:
         machine_type: [single-gpu, multi-gpu]
     runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }}
     container:
-      image: huggingface/transformers-all-latest-gpu
+      image: huggingface/internal-transformers-all-latest-gpu-tokenizers-main
       options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     steps:
       - name: NVIDIA-SMI
         run: |
@@ -57,8 +63,11 @@ jobs:
         machine_type: [single-gpu, multi-gpu]
     runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }}
     container:
-      image: huggingface/transformers-all-latest-gpu
+      image: huggingface/internal-transformers-all-latest-gpu-tokenizers-main
       options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     outputs:
       matrix: ${{ steps.set-matrix.outputs.matrix }}
     steps:
@@ -93,8 +102,11 @@ jobs:
         machine_type: [single-gpu]
     runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }}
     container:
-      image: huggingface/transformers-all-latest-gpu
+      image: huggingface/internal-transformers-all-latest-gpu-tokenizers-main
       options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     needs: setup
     steps:
       - name: Echo folder ${{ matrix.folders }}
@@ -146,8 +158,11 @@ jobs:
         machine_type: [multi-gpu]
     runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }}
     container:
-      image: huggingface/transformers-all-latest-gpu
+      image: huggingface/internal-transformers-all-latest-gpu-tokenizers-main
       options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     needs: setup
     steps:
       - name: Echo folder ${{ matrix.folders }}
@@ -194,8 +209,11 @@ jobs:
     name: Examples directory
     runs-on: [self-hosted, single-gpu-docker]
     container:
-      image: huggingface/transformers-all-latest-gpu
+      image: huggingface/internal-transformers-all-latest-gpu-tokenizers-main
       options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     needs: setup
     steps:
       - name: Update clone
@@ -237,8 +255,11 @@ jobs:
         machine_type: [single-gpu, multi-gpu]
     runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }}
     container:
-      image: huggingface/transformers-pytorch-gpu
-      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      image: huggingface/internal-transformers-pytorch-gpu-tokenizers-main
+      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     needs: setup
     steps:
       - name: Update clone
@@ -281,8 +302,11 @@ jobs:
         machine_type: [single-gpu, multi-gpu]
     runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }}
     container:
-      image: huggingface/transformers-tensorflow-gpu
-      options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      image: huggingface/internal-transformers-tensorflow-gpu-tokenizers-main
+      options: --gpus 0 --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     needs: setup
     steps:
       - name: Update clone
@@ -327,8 +351,11 @@ jobs:
     runs-on: ${{ format('{0}-{1}', matrix.machine_type, 'docker') }}
     needs: setup
     container:
-      image: huggingface/transformers-pytorch-deepspeed-latest-gpu
+      image: huggingface/internal-transformers-pytorch-deepspeed-latest-gpu-tokenizers-main
       options: --gpus all --shm-size "16gb" --ipc host -v /mnt/cache/.cache/huggingface:/mnt/cache/
+      credentials:
+        username: ${{ secrets.DOCKERHUB_USERNAME }}
+        password: ${{ secrets.DOCKERHUB_PASSWORD }}
     steps:
       - name: Update clone
         working-directory: /workspace/transformers
@@ -400,7 +427,7 @@ jobs:
         env:
           CI_SLACK_BOT_TOKEN: ${{ secrets.CI_SLACK_BOT_TOKEN }}
           CI_SLACK_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID }}
-          CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }}
+          CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
           CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }}
           CI_SLACK_REPORT_CHANNEL_ID: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }}
           CI_EVENT: scheduled

diff --git a/setup.py b/setup.py
@@ -160,7 +160,7 @@
     "tf2onnx",
     "timeout-decorator",
     "timm",
-    "tokenizers>=0.11.1,!=0.11.3,<0.14",
+    "tokenizers",
     "torch>=1.7,!=1.12.0",
     "torchaudio",
     "pyctcdecode>=0.3.0",
@@ -333,8 +333,7 @@ def run(self):
     extras["testing"]
     + extras["torch"]
     + extras["sentencepiece"]
-    + extras["tokenizers"]
-    + extras["torch-speech"]
+    + extras["tokenizers"] + extras["torch-speech"]
     + extras["vision"]
     + extras["integrations"]
     + extras["timm"]

diff --git a/src/transformers/dependency_versions_table.py b/src/transformers/dependency_versions_table.py
@@ -66,7 +66,7 @@
     "tf2onnx": "tf2onnx",
     "timeout-decorator": "timeout-decorator",
     "timm": "timm",
-    "tokenizers": "tokenizers>=0.11.1,!=0.11.3,<0.14",
+    "tokenizers": "tokenizers",
     "torch": "torch>=1.7,!=1.12.0",
     "torchaudio": "torchaudio",
     "pyctcdecode": "pyctcdecode>=0.3.0",