feat: added en-core-web-sm as en explicit dependency

aidial_interceptors_sdk/examples/chat_completion/pii_anonymiser/spacy_anonymizer.py

@@ -1,3 +1,4 @@
+import logging
 import re
 from collections import defaultdict
 from functools import cache
@@ -22,11 +23,19 @@
     "PRODUCT",
 ]
 
+_log = logging.getLogger(__name__)
+
 
 @cache
 def _get_pipeline(model: str) -> Language:
-    download_model(model)
-    return load_model(model)
+    try:
+        return load_model(model)
+    except Exception as e:
+        _log.warning(
+            f"Failed to load spaCy model {model!r}: {str(e)}\nDownloading the model..."
+        )
+        download_model(model)
+        return load_model(model)
 
 
 # Preemptively load the default model on the server start-up

pyproject.toml

@@ -31,10 +31,17 @@ aidial-sdk = { version = "^0.15.0", extras = ["telemetry"] }
 aiostream = { version = "^0.6.2", optional = true }
 pillow = { version = "^10.4.0", optional = true }
 numpy = { version = "^1.26.1", optional = true }
-spacy = { version = "^3.7.5", optional = true }
+spacy = { version = "3.7.5", optional = true }
+
+[tool.poetry.dependencies.en_core_web_sm]
+# Pinning particular combination of Spacy and en_core_web_sm versions:
+# https://github.com/explosion/spaCy/issues/13690#issuecomment-2487873386
+# otherwise, have a change of running into 403 error in runtime.
+url = "https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.7.1/en_core_web_sm-3.7.1-py3-none-any.whl"
+optional = true
 
 [tool.poetry.extras]
-examples = ["aiostream", "pillow", "numpy", "spacy"]
+examples = ["aiostream", "pillow", "numpy", "spacy", "en_core_web_sm"]
 
 [tool.poetry.group.test.dependencies]
 pytest = "7.4.0"