Unconditional HF_HUB_OFFLINE

.github/workflows/job_jax_layer_tests.yml

@@ -75,6 +75,7 @@ jobs:
         run: |
           echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
+          echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
 
       - name: Install OpenVINO dependencies (mac)
         if: runner.os == 'macOS'

.github/workflows/job_jax_models_tests.yml

@@ -77,6 +77,7 @@ jobs:
         run: |
           echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
+          echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
 
       - name: Extract OpenVINO packages and tests
         run: pigz -dc openvino_tests.tar.gz | tar -xf - -v

.github/workflows/job_onnx_models_tests.yml

@@ -78,6 +78,7 @@ jobs:
         run: |
           echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "HUGGINGFACE_HUB_CACHE=$HF_HUB_CACHE" >> "$GITHUB_ENV"
+          echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
           echo "MODELS_SHARE_PATH=/mount/testdata$((GITHUB_RUN_NUMBER % NUMBER_OF_REPLICAS))" >> "$GITHUB_ENV"
 
       # Issue 148922

.github/workflows/job_pytorch_fx_layer_tests.yml

@@ -74,6 +74,7 @@ jobs:
         run: |
           Add-Content -Path $env:GITHUB_ENV -Value "HF_HUB_CACHE=C:\\mount\\caches\\huggingface"
           Add-Content -Path $env:GITHUB_ENV -Value "HUGGINGFACE_HUB_CACHE=C:\\mount\\caches\\huggingface"
+          Add-Content -Path $env:GITHUB_ENV -Value "HF_HUB_OFFLINE=1"
 
       - name: Install OpenVINO dependencies (mac)
         if: runner.os == 'macOS'

.github/workflows/job_pytorch_layer_tests.yml

@@ -75,6 +75,7 @@ jobs:
         run: |
           echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
+          echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
       - name: Install OpenVINO dependencies (mac)
         if: runner.os == 'macOS'
         run: brew install pigz

.github/workflows/job_pytorch_models_tests.yml

@@ -75,6 +75,7 @@ jobs:
         run: |
           echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
+          echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
 
       - name: Extract OpenVINO artifacts
         run: pigz -dc openvino_tests.tar.gz | tar -xf - -v
@@ -118,6 +119,7 @@ jobs:
           TYPE: ${{ inputs.model_scope == 'precommit' && 'precommit' || 'nightly' }}
           TEST_DEVICE: CPU
           OP_REPORT_FILE: ${{ env.INSTALL_TEST_DIR }}/TEST-torch_unsupported_ops.log
+          HF_HUB_OFFLINE: 0  # Can't use offline mode - these tests use `model_info` call, which can't be cached
 
       - name: PagedAttention Test
         if: ${{ inputs.model_scope == 'precommit' }}

.github/workflows/job_tensorflow_layer_tests.yml

@@ -80,6 +80,7 @@ jobs:
         run: |
           echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
+          echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
 
       - name: Install OpenVINO dependencies (mac)
         if: ${{ runner.os == 'macOS' }}

.github/workflows/job_tensorflow_models_tests.yml

@@ -76,6 +76,7 @@ jobs:
           echo "HF_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "HUGGINGFACE_HUB_CACHE=${{ runner.os == 'Linux' && env.HF_HUB_CACHE_LIN || env.HF_HUB_CACHE_WIN }}" >> "$GITHUB_ENV"
           echo "TFHUB_CACHE_DIR=/mount/testdata$((GITHUB_RUN_NUMBER % NUMBER_OF_REPLICAS))/tfhub_models" >> "$GITHUB_ENV"
+          echo "HF_HUB_OFFLINE=1" >> "$GITHUB_ENV"
 
       - name: Extract OpenVINO artifacts (Linux and macOS)
         run: pigz -dc openvino_tests.tar.gz | tar -xf - -v

tests/model_hub_tests/models_hub_common/utils.py

@@ -155,7 +155,18 @@ def print_stat(s: str, value: float):
     print(s.format(round_num(value)))
 
 
-def retry(max_retries=3, exceptions=(Exception,), delay=None):
+def retry(max_retries=3, exceptions=(Exception,), delay=None, exponential_backoff=False, backoff_multiplier=2, max_delay=None):
+    """
+    Retry decorator with optional exponential backoff.
+
+    Args:
+        max_retries: Maximum number of retry attempts
+        exceptions: Tuple of exception types to catch and retry on
+        delay: Base delay in seconds between retries
+        exponential_backoff: If True, use exponential backoff instead of fixed delay
+        backoff_multiplier: Multiplier for exponential backoff (default: 2)
+        max_delay: Maximum delay cap for exponential backoff
+    """
     def retry_decorator(func):
         @functools.wraps(func)
         def wrapper(*args, **kwargs):
@@ -165,7 +176,18 @@ def wrapper(*args, **kwargs):
                 except exceptions as e:
                     print(f"Attempt {attempt + 1} of {max_retries} failed: {e}")
                     if attempt < max_retries - 1 and delay is not None:
-                        time.sleep(delay)
+                        if exponential_backoff:
+                            # Calculate exponential backoff delay
+                            backoff_delay = delay * (backoff_multiplier ** attempt)
+                            # Apply max_delay cap if specified
+                            if max_delay is not None:
+                                backoff_delay = min(backoff_delay, max_delay)
+                            print(f"Waiting {backoff_delay:.2f} seconds before retry")
+                            time.sleep(backoff_delay)
+                        else:
+                            # Use fixed delay
+                            print(f"Waiting {delay} seconds before retry")
+                            time.sleep(delay)
                     else:
                         raise e
         return wrapper

tests/model_hub_tests/pytorch/test_hf_transformers.py

@@ -55,7 +55,7 @@ def setup_class(self):
         url = "http://images.cocodataset.org/val2017/000000039769.jpg"
         self.image = Image.open(requests.get(url, stream=True).raw)
 
-    @retry(3, exceptions=(OSError,), delay=1)
+    @retry(10, exceptions=(OSError,), delay=5, exponential_backoff=True, backoff_multiplier=2, max_delay=300)
     def load_model(self, name, type):
         name, _, name_suffix = name.partition(':')