ci: create e2e environment; stop testing spacy in unit tests

Author: anakin87

.github/workflows/e2e.yml

@@ -36,7 +36,7 @@ jobs:
       run: pip install hatch==${{ env.HATCH_VERSION }}
 
     - name: Run tests
-      run: hatch run test:e2e
+      run: hatch run e2e:test
 
     - name: Send event to Datadog
       if: failure() && github.event_name == 'schedule'

haystack/components/extractors/named_entity_extractor.py

@@ -19,8 +19,8 @@
     from transformers import Pipeline as HfPipeline
 
 with LazyImport(message="Run 'pip install spacy'") as spacy_import:
-    import spacy
-    from spacy import Language as SpacyPipeline
+    import spacy  # pylint: disable=import-error
+    from spacy import Language as SpacyPipeline  # pylint: disable=import-error
 
 
 class NamedEntityExtractorBackend(Enum):

pyproject.toml

@@ -81,7 +81,7 @@ format = "ruff format {args}"
 format-check = "ruff format --check {args}"
 
 [tool.hatch.envs.test]
-extra-dependencies = [
+dependencies = [
   "numpy>=2", # Haystack is compatible both with numpy 1.x and 2.x, but we test with 2.x
   "numba>=0.54.0", # This pin helps uv resolve the dependency tree. See https://github.com/astral-sh/uv/issues/7881
 
@@ -95,13 +95,6 @@ extra-dependencies = [
   "openai-whisper>=20231106",                  # LocalWhisperTranscriber
   "arrow>=1.3.0",                              # Jinja2TimeExtension
 
-  # NamedEntityExtractor
-  "spacy>=3.8,<3.9",
-  "spacy-curated-transformers>=0.2,<=0.3",
-  "en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.8.0/en_core_web_trf-3.8.0-py3-none-any.whl",
-  # spacy requires thinc, which depends on blis. We pin blis because version 1.2.1 does not have wheels for python 3.9
-  # and compiling it from source takes much time.
-  "blis<1.2.1; python_version < '3.10'",
 
   # Converters
   "pypdf",                            # PyPDFToDocument
@@ -161,14 +154,27 @@ extra-dependencies = [
 ]
 
 [tool.hatch.envs.test.scripts]
-e2e = "pytest e2e"
 unit = 'pytest --cov-report xml:coverage.xml --cov="haystack" -m "not integration" {args:test}'
 integration = 'pytest --maxfail=5 -m "integration" {args:test}'
 integration-mac = 'pytest --maxfail=5 -m "integration" -k "not tika" {args:test}'
 integration-windows = 'pytest --maxfail=5 -m "integration" -k "not tika" {args:test}'
 types = "mypy --install-types --non-interactive --cache-dir=.mypy_cache/ {args:haystack}"
 lint = "pylint -ry -j 0 {args:haystack}"
 
+[tool.hatch.envs.e2e]
+template = "test"
+extra-dependencies = [
+  # NamedEntityExtractor
+  "spacy>=3.8,<3.9",
+  "en-core-web-trf @ https://github.com/explosion/spacy-models/releases/download/en_core_web_trf-3.8.0/en_core_web_trf-3.8.0-py3-none-any.whl",
+  # spacy requires thinc, which depends on blis. We pin blis because version 1.2.1 does not have wheels for python 3.9
+  # and compiling it from source takes much time.
+  "blis<1.2.1; python_version < '3.10'",
+]
+
+[tool.hatch.envs.e2e.scripts]
+test = "pytest e2e"
+
 [tool.hatch.envs.readme]
 installer = "uv"
 detached = true                         # To avoid installing the dependencies from the default environment

test/components/extractors/test_named_entity_extractor.py

@@ -1,6 +1,11 @@
 # SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
 #
 # SPDX-License-Identifier: Apache-2.0
+
+# Note: We do not test the Spacy backend in this module.
+# Spacy is not installed in the test environment to keep the CI fast.
+# We test the Spacy backend in e2e/pipelines/test_named_entity_extractor.py.
+
 from haystack.utils.auth import Secret
 import pytest
 
@@ -17,10 +22,6 @@ def test_named_entity_extractor_backend():
 
     _ = NamedEntityExtractor(backend="hugging_face", model="dslim/bert-base-NER")
 
-    _ = NamedEntityExtractor(backend=NamedEntityExtractorBackend.SPACY, model="en_core_web_sm")
-
-    _ = NamedEntityExtractor(backend="spacy", model="en_core_web_sm")
-
     with pytest.raises(ComponentError, match=r"Invalid backend"):
         NamedEntityExtractor(backend="random_backend", model="dslim/bert-base-NER")