Skip to content

Commit d3a339e

Browse files
authored
Allow editable install (#155)
* Get path for editable install * Detect and work with existing editable installs * Fix Windows * Better find Python executable on Windows * Use distribution name, not package name, to check installed metadata * Refactor editable tests * Fix linting * Don't try to generate gcov for editable install * Correct working dir for generating gcov * Mention editable install in README
1 parent a6b0283 commit d3a339e

File tree

5 files changed

+124
-42
lines changed

5 files changed

+124
-42
lines changed

README.md

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,17 @@
33
## A developer tool for scientific Python libraries
44

55
Developers need to memorize a whole bunch of magic command-line incantations.
6-
And these incantations change from time to time!
7-
Typically, their lives are made simpler by a Makefile, but Makefiles can be convoluted, are not written in Python, and are hard to extend.
8-
The rationale behind `spin` is therefore to provide a simple interface for common development tasks.
6+
These incantations may also change over time.
7+
Often, Makefiles are used to provide aliases, but Makefiles can be convoluted, are not written in Python, and are hard to extend.
8+
The goal of `spin` is therefore to provide a simple, user-friendly, extendable interface for common development tasks.
99
It comes with a few common build commands out the box, but can easily be customized per project.
1010

1111
As a curiosity: the impetus behind developing the tool was the mass migration of scientific Python libraries (SciPy, scikit-image, and NumPy, etc.) to Meson, after distutils was deprecated.
1212
When many of the build and installation commands changed, it made sense to abstract away the nuisance of having to re-learn them.
1313

14+
_Note:_ We now have experimental builds for editable installs.
15+
Most of the Meson commands listed below should work "out of the box" for those.
16+
1417
## Installation
1518

1619
```
@@ -92,7 +95,6 @@ Available as `spin.cmds.meson.*`.
9295
docs 📖 Build Sphinx documentation
9396
gdb 👾 Execute a Python snippet with GDB
9497
lldb 👾 Execute a Python snippet with LLDB
95-
install 💽 Build and install package using pip.
9698
```
9799

98100
### [Build](https://pypa-build.readthedocs.io/en/stable/) (PEP 517 builder)

example_pkg/pyproject.toml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ package = 'example_pkg'
2929
"spin.cmds.meson.build",
3030
"spin.cmds.meson.test",
3131
"spin.cmds.build.sdist",
32-
"spin.cmds.pip.install",
3332
]
3433
"Documentation" = [
3534
"spin.cmds.meson.docs"
@@ -45,7 +44,7 @@ package = 'example_pkg'
4544
"spin.cmds.meson.lldb"
4645
]
4746
"Extensions" = [".spin/cmds.py:example"]
48-
"Install" = [
47+
"Pip" = [
4948
"spin.cmds.pip.install"
5049
]
5150

spin/cmds/meson.py

Lines changed: 96 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,32 @@ def _meson_cli():
3737
return [meson_cli]
3838

3939

40+
def _editable_install_path(distname):
41+
import importlib_metadata
42+
43+
try:
44+
dist = importlib_metadata.Distribution.from_name(distname)
45+
except importlib_metadata.PackageNotFoundError:
46+
return None
47+
48+
if getattr(dist.origin.dir_info, "editable", False):
49+
if sys.platform == "win32":
50+
return dist.origin.url.removeprefix("file:///")
51+
else:
52+
return dist.origin.url.removeprefix("file://")
53+
else:
54+
return None
55+
56+
57+
def _is_editable_install(distname):
58+
return _editable_install_path(distname) is not None
59+
60+
61+
def _is_editable_install_of_same_source(distname):
62+
editable_path = _editable_install_path(distname)
63+
return editable_path and os.path.samefile(_editable_install_path(distname), ".")
64+
65+
4066
def _set_pythonpath(quiet=False):
4167
"""Set first entry of PYTHONPATH to site packages directory.
4268
@@ -48,24 +74,27 @@ def _set_pythonpath(quiet=False):
4874
env = os.environ
4975

5076
cfg = get_config()
51-
package = cfg.get("tool.spin.package", None)
52-
if package:
53-
import importlib_metadata
54-
55-
try:
56-
dist = importlib_metadata.Distribution.from_name(package)
57-
if getattr(dist.origin.dir_info, "editable", False):
77+
distname = cfg.get("project.name", None)
78+
if distname:
79+
if _is_editable_install(distname):
80+
if _is_editable_install_of_same_source(distname):
81+
if not (quiet):
82+
click.secho(
83+
"Editable install of same source directory detected; not setting PYTHONPATH",
84+
fg="bright_red",
85+
)
86+
return ""
87+
else:
88+
# Ignoring the quiet flag, because picking up the wrong package is problematic
5889
click.secho(
59-
f"Warning! An editable installation of `{package}` was detected.",
90+
f"Warning! Editable install of `{distname}`, from a different source location, detected.",
6091
fg="bright_red",
6192
)
6293
click.secho("Spin commands will pick up that version.", fg="bright_red")
6394
click.secho(
64-
f"Try removing the other installation with `pip uninstall {package}`.",
95+
f"Try removing the other installation by switching to its source and running `pip uninstall {distname}`.",
6596
fg="bright_red",
6697
)
67-
except importlib_metadata.PackageNotFoundError:
68-
pass
6998

7099
if "PYTHONPATH" in env:
71100
env["PYTHONPATH"] = f"{site_packages}{os.pathsep}{env['PYTHONPATH']}"
@@ -81,6 +110,15 @@ def _set_pythonpath(quiet=False):
81110

82111

83112
def _get_site_packages():
113+
try:
114+
cfg = get_config()
115+
distname = cfg.get("project.name", None)
116+
if _is_editable_install_of_same_source(distname):
117+
return ""
118+
except RuntimeError:
119+
# Probably not running in click
120+
pass
121+
84122
candidate_paths = []
85123
for root, dirs, _files in os.walk(install_dir):
86124
for subdir in dirs:
@@ -162,7 +200,7 @@ def _check_coverage_tool_installation(coverage_type: GcovReportFormat):
162200
# First check the presence of a valid build
163201
if not (os.path.exists(build_dir)):
164202
raise click.ClickException(
165-
"`build` folder not found, cannot generate coverage reports. "
203+
f"`{build_dir}` folder not found, cannot generate coverage reports. "
166204
"Generate coverage artefacts by running `spin test --gcov`"
167205
)
168206

@@ -213,6 +251,16 @@ def build(meson_args, jobs=None, clean=False, verbose=False, gcov=False, quiet=F
213251
214252
CFLAGS="-O0 -g" spin build
215253
"""
254+
cfg = get_config()
255+
distname = cfg.get("project.name", None)
256+
if distname and _is_editable_install_of_same_source(distname):
257+
if not quiet:
258+
click.secho(
259+
"Editable install of same source detected; skipping build",
260+
fg="bright_red",
261+
)
262+
return
263+
216264
meson_args = list(meson_args)
217265

218266
if gcov:
@@ -371,6 +419,14 @@ def test(
371419
For more, see `pytest --help`.
372420
""" # noqa: E501
373421
cfg = get_config()
422+
distname = cfg.get("project.name", None)
423+
424+
if gcov and distname and _is_editable_install_of_same_source(distname):
425+
click.secho(
426+
"Error: cannot generate coverage report for editable installs",
427+
fg="bright_red",
428+
)
429+
raise SystemExit(-1)
374430

375431
build_cmd = _get_configured_command("build")
376432
if build_cmd:
@@ -389,6 +445,8 @@ def test(
389445
sys.exit(1)
390446

391447
site_path = _set_pythonpath()
448+
if site_path:
449+
print(f'$ export PYTHONPATH="{site_path}"')
392450

393451
# Sanity check that library built properly
394452
#
@@ -428,13 +486,17 @@ def test(
428486
f"--cov={package}",
429487
]
430488

431-
print(f'$ export PYTHONPATH="{site_path}"')
432-
433489
if sys.version_info[:2] >= (3, 11):
434490
cmd = [sys.executable, "-P", "-m", "pytest"]
435491
else:
436492
cmd = ["pytest"]
437-
p = _run(cmd + list(pytest_args))
493+
494+
cwd = os.getcwd()
495+
pytest_p = _run(
496+
cmd + ([f"--rootdir={site_path}"] if site_path else []) + list(pytest_args),
497+
cwd=site_path,
498+
)
499+
os.chdir(cwd)
438500

439501
if gcov:
440502
# Verify the tools are present
@@ -471,7 +533,7 @@ def test(
471533
fg="bright_green",
472534
)
473535

474-
raise SystemExit(p.returncode)
536+
raise SystemExit(pytest_p.returncode)
475537

476538

477539
@click.command()
@@ -545,7 +607,8 @@ def ipython(ctx, ipython_args):
545607
ctx.invoke(build_cmd)
546608

547609
p = _set_pythonpath()
548-
print(f'💻 Launching IPython with PYTHONPATH="{p}"')
610+
if p:
611+
print(f'💻 Launching IPython with PYTHONPATH="{p}"')
549612
_run(["ipython", "--ignore-cwd"] + list(ipython_args), replace=True)
550613

551614

@@ -570,9 +633,11 @@ def shell(ctx, shell_args=[]):
570633
ctx.invoke(build_cmd)
571634

572635
p = _set_pythonpath()
636+
if p:
637+
print(f'💻 Launching shell with PYTHONPATH="{p}"')
638+
573639
shell = os.environ.get("SHELL", "sh")
574640
cmd = [shell] + list(shell_args)
575-
print(f'💻 Launching shell with PYTHONPATH="{p}"')
576641
print("⚠ Change directory to avoid importing source instead of built package")
577642
print("⚠ Ensure that your ~/.shellrc does not unset PYTHONPATH")
578643
_run(cmd, replace=True)
@@ -596,6 +661,9 @@ def python(ctx, python_args):
596661
ctx.invoke(build_cmd)
597662

598663
p = _set_pythonpath()
664+
if p:
665+
print(f'🐍 Launching Python with PYTHONPATH="{p}"')
666+
599667
v = sys.version_info
600668
if (v.major < 3) or (v.major == 3 and v.minor < 11):
601669
print("We're sorry, but this feature only works on Python 3.11 and greater 😢")
@@ -613,9 +681,7 @@ def python(ctx, python_args):
613681
print("import sys; del(sys.path[0])")
614682
sys.exit(-1)
615683

616-
print(f'🐍 Launching Python with PYTHONPATH="{p}"')
617-
618-
_run(["/usr/bin/env", "python", "-P"] + list(python_args), replace=True)
684+
_run([sys.executable, "-P"] + list(python_args), replace=True)
619685

620686

621687
@click.command(context_settings={"ignore_unknown_options": True})
@@ -762,10 +828,15 @@ def docs(ctx, sphinx_target, clean, first_build, jobs, sphinx_gallery_plot):
762828
f"$ export SPHINXOPTS={os.environ['SPHINXOPTS']}", bold=True, fg="bright_blue"
763829
)
764830

765-
os.environ["PYTHONPATH"] = f'{site_path}{os.sep}:{os.environ.get("PYTHONPATH", "")}'
766-
click.secho(
767-
f"$ export PYTHONPATH={os.environ['PYTHONPATH']}", bold=True, fg="bright_blue"
768-
)
831+
if site_path:
832+
os.environ["PYTHONPATH"] = (
833+
f'{site_path}{os.sep}:{os.environ.get("PYTHONPATH", "")}'
834+
)
835+
click.secho(
836+
f"$ export PYTHONPATH={os.environ['PYTHONPATH']}",
837+
bold=True,
838+
fg="bright_blue",
839+
)
769840
_run(["make", "-C", doc_dir, sphinx_target], replace=True)
770841

771842

spin/tests/test_build_cmds.py

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -76,17 +76,6 @@ def test_run_stdout():
7676
), f"`spin run` stdout did not yield version, but {stdout(p)}"
7777

7878

79-
def test_editable_conflict():
80-
"""Do we warn when a conflicting editable install is present?"""
81-
try:
82-
run(["pip", "install", "--quiet", "-e", "."])
83-
assert "Warning! An editable installation" in stdout(
84-
spin("run", "ls")
85-
), "Failed to detect and warn about editable install"
86-
finally:
87-
run(["pip", "uninstall", "--quiet", "-y", "example_pkg"])
88-
89-
9079
# Detecting whether a file is executable is not that easy on Windows,
9180
# as it seems to take into consideration whether that file is associated as an executable.
9281
@skip_on_windows

spin/tests/test_editable.py

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
import pytest
2+
from testutil import spin, stdout
3+
4+
from spin.cmds.util import run
5+
6+
7+
@pytest.fixture
8+
def editable_install():
9+
run(["pip", "install", "--quiet", "--no-build-isolation", "-e", "."])
10+
yield
11+
run(["pip", "uninstall", "--quiet", "-y", "example_pkg"])
12+
13+
14+
def test_detect_editable(editable_install):
15+
assert "Editable install of same source detected" in stdout(
16+
spin("build")
17+
), "Failed to detect and warn about editable install"
18+
19+
20+
def test_editable_tests(editable_install):
21+
spin("test")

0 commit comments

Comments
 (0)