Fallback to memory cache when disk is not available (#8718)

* Fallback to memory cache when disk is not available

* Update dspy/clients/__init__.py

Co-authored-by: Copilot <[email protected]>

---------

Co-authored-by: Copilot <[email protected]>

Author: TomeHirata

dspy/clients/__init__.py

@@ -31,27 +31,47 @@ def configure_cache(
         memory_max_entries: The maximum number of entries in the in-memory cache.
     """
 
-    import dspy
-
-    dspy.cache = Cache(
+    DSPY_CACHE = Cache(
         enable_disk_cache,
         enable_memory_cache,
         disk_cache_dir,
         disk_size_limit_bytes,
         memory_max_entries,
     )
 
+    import dspy
+    # Update the reference to point to the new cache
+    dspy.cache = DSPY_CACHE
+
 
 litellm.telemetry = False
 litellm.cache = None  # By default we disable LiteLLM cache and use DSPy on-disk cache.
 
-DSPY_CACHE = Cache(
-    enable_disk_cache=True,
-    enable_memory_cache=True,
-    disk_cache_dir=DISK_CACHE_DIR,
-    disk_size_limit_bytes=DISK_CACHE_LIMIT,
-    memory_max_entries=1000000,
-)
+def _get_dspy_cache():
+    disk_cache_dir = os.environ.get("DSPY_CACHEDIR") or os.path.join(Path.home(), ".dspy_cache")
+    disk_cache_limit = int(os.environ.get("DSPY_CACHE_LIMIT", 3e10))
+
+    try:
+        _dspy_cache = Cache(
+            enable_disk_cache=True,
+            enable_memory_cache=True,
+            disk_cache_dir=disk_cache_dir,
+            disk_size_limit_bytes=disk_cache_limit,
+            memory_max_entries=1000000,
+        )
+    except Exception as e:
+        # If cache creation fails (e.g., in AWS Lambda), create a memory-only cache
+        logger.warning("Failed to initialize disk cache, falling back to memory-only cache: %s", e)
+        _dspy_cache = Cache(
+            enable_disk_cache=False,
+            enable_memory_cache=True,
+            disk_cache_dir=disk_cache_dir,
+            disk_size_limit_bytes=disk_cache_limit,
+            memory_max_entries=1000000,
+        )
+    return _dspy_cache
+
+DSPY_CACHE = _get_dspy_cache()
 
 if "LITELLM_LOCAL_MODEL_COST_MAP" not in os.environ:
     # Accessed at run time by litellm; i.e., fine to keep after import

tests/clients/test_cache.py

@@ -1,3 +1,4 @@
+import os
 from dataclasses import dataclass
 from unittest.mock import patch
 
@@ -317,3 +318,29 @@ def test_function(**kwargs):
             )
             is not None
         )
+
+
+def test_cache_fallback_on_restricted_environment():
+    """Test that DSPy gracefully falls back to memory-only cache when disk cache fails."""
+    old_env = os.environ.get("DSPY_CACHEDIR")
+    try:
+        # Set an invalid cache directory that can't be created
+        os.environ["DSPY_CACHEDIR"] = "/dev/null/invalid_path"
+
+        import dspy
+        from dspy.clients import _get_dspy_cache
+
+        dspy.cache = _get_dspy_cache()
+
+        # Cache should work with memory-only fallback despite invalid disk path
+        test_request = {"model": "test", "prompt": "hello"}
+        dspy.cache.put(test_request, "fallback_result")
+        result = dspy.cache.get(test_request)
+
+        assert result == "fallback_result", "Memory cache fallback should work"
+
+    finally:
+        if old_env is None:
+            os.environ.pop("DSPY_CACHEDIR", None)
+        else:
+            os.environ["DSPY_CACHEDIR"] = old_env