diff --git a/.circleci/create_circleci_config.py b/.circleci/create_circleci_config.py
index 26f254d141b7..de0646af2dbf 100644
--- a/.circleci/create_circleci_config.py
+++ b/.circleci/create_circleci_config.py
@@ -232,12 +232,18 @@ def to_dict(self):
                     "command": "cp -r /test_data/* . 2>/dev/null || true; python3 utils/fetch_hub_objects_for_ci.py",
                 }
             },
-            {
-                "run": {
-                    "name": "download and unzip hub cache",
-                    "command": 'curl -L -o huggingface-cache.tar.gz https://huggingface.co/datasets/hf-internal-testing/hf_hub_cache/resolve/main/huggingface-cache.tar.gz && apt-get install pigz && tar --use-compress-program="pigz -d -p 8" -xf huggingface-cache.tar.gz && mv -n hub/* /root/.cache/huggingface/hub/ && ls -la /root/.cache/huggingface/hub/',
-                }
-            },
+            *(
+                [
+                    {
+                        "run": {
+                            "name": "download and unzip hub cache",
+                            "command": 'curl -L -o huggingface-cache.tar.gz https://huggingface.co/datasets/hf-internal-testing/hf_hub_cache/resolve/main/huggingface-cache.tar.gz && apt-get install pigz && tar --use-compress-program="pigz -d -p 8" -xf huggingface-cache.tar.gz && mv -n hub/* /root/.cache/huggingface/hub/ && ls -la /root/.cache/huggingface/hub/',
+                        }
+                    }
+                ]
+                if self.job_name == "pipelines_torch"
+                else []
+            ),
             {
                 "run": {
                     "name": "Run tests",