From 716b60c0fa16140d484a0ea288a91eb810280d85 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Wed, 8 May 2024 00:58:46 +0200 Subject: [PATCH 1/3] catch up with RU --- setup.py | 222 ++++++++++++++++++++----------------------------------- 1 file changed, 81 insertions(+), 141 deletions(-) diff --git a/setup.py b/setup.py index 0cb085d4c..c0c0f9b0c 100644 --- a/setup.py +++ b/setup.py @@ -8,55 +8,35 @@ ''' Setup script, only usable via pip. ''' -import re import os -import sys -import glob -import shutil import subprocess as sp +from glob import glob from setuptools import setup, Command, find_namespace_packages # ------------------------------------------------------------------------------ +# base = 'saga' name = 'radical.%s' % base mod_root = 'src/radical/%s/' % base -# ------------------------------------------------------------------------------ -# -# pip warning: -# "In-tree builds are now default. pip 22.1 will enforce this behaviour change. -# A possible replacement is to remove the --use-feature=in-tree-build flag." -# -# With this change we need to make sure to clean out all temporary files from -# the src tree. Specifically create (and thus need to clean) -# - VERSION -# - SDIST -# - the sdist file itself (a tarball) -# -# `pip install` (or any other direct or indirect invocation of `setup.py`) will -# in fact run `setup.py` multiple times: one on the top level, and internally -# again with other arguments to build sdist and bwheel packages. We must *not* -# clean out temporary files in those internal runs as that would invalidate the -# install. -# -# We thus introduce an env variable `SDIST_LEVEL` which allows us to separate -# internal calls from the top level invocation - we only clean on the latter -# (see end of this file). -sdist_level = int(os.environ.get('SDIST_LEVEL', 0)) -os.environ['SDIST_LEVEL'] = str(sdist_level + 1) +scripts = list(glob('bin/*')) +root = os.path.dirname(__file__) or '.' +readme = open("%s/README.md" % root, encoding='utf-8').read() +descr = "A light-weight access layer for distributed compute infrastructure" +keywords = ['radical', 'cybertools', 'utilities', 'saga', 'job', 'compute'] -root = os.path.dirname(__file__) or '.' +share = 'share/%s' % name +data = [('%s/examples' % share, glob('examples/*.{py,cfg,json,sh}')), +] # ------------------------------------------------------------------------------ # def sh_callout(cmd): - p = sp.Popen(cmd, stdout=sp.PIPE, stderr=sp.PIPE, shell=True) - stdout, stderr = p.communicate() ret = p.returncode return stdout, stderr, ret @@ -64,17 +44,6 @@ def sh_callout(cmd): # ------------------------------------------------------------------------------ # -# versioning mechanism: -# -# - version: 1.2.3 - is used for installation -# - version_detail: v1.2.3-9-g0684b06 - is used for debugging -# - version is read from VERSION file in root, which then is copied to -# module dir, and is getting installed from there. -# - version_detail is derived from the git tag, and only available when -# installed from git. That is stored in mod_root/VERSION in the install -# tree. -# - The VERSION file is used to provide the runtime version information. -# def get_version(_mod_root): ''' a VERSION file containes the version strings is created in mod_root, @@ -82,96 +51,77 @@ def get_version(_mod_root): information. ''' + _out = None + _err = None + _ret = None try: - + _version_path = '%s/%s/VERSION' % (root, _mod_root) _version_base = None + _version_short = None + _version_branch = None + _version_tag = None _version_detail = None - _sdist_name = None - # get version from './VERSION' + # get `version_base` from distribution's 'VERSION' file with open('%s/VERSION' % root, 'r', encoding='utf-8') as fin: _version_base = fin.readline().strip() - # attempt to get version detail information from git - # We only do that though if we are in a repo root dir, - # ie. if 'git rev-parse --show-prefix' returns an empty string -- - # otherwise we get confused if the ve lives beneath another repository, - # and the pip version used uses an install tmp dir in the ve space - # instead of /tmp (which seems to happen with some pip/setuptools - # versions). - out, _, ret = sh_callout( - 'cd %s ; ' - 'test -z `git rev-parse --show-prefix` || exit -1; ' - 'tag=`git describe --tags --always` 2>/dev/null ; ' - 'branch=`git branch | grep -e "^*" | cut -f 2- -d " "` 2>/dev/null ; ' - 'echo $tag@$branch' % root) - _version_detail = out.strip() - _version_detail = _version_detail.decode() - _version_detail = _version_detail.replace('detached from ', 'detached-') - - # remove all non-alphanumeric (and then some) chars - _version_detail = re.sub('[/ ]+', '-', _version_detail) - _version_detail = re.sub('[^a-zA-Z0-9_+@.-]+', '', _version_detail) - - if ret != 0 or \ - _version_detail == '@' or \ - 'git-error' in _version_detail or \ - 'not-a-git-repo' in _version_detail or \ - 'not-found' in _version_detail or \ - 'fatal' in _version_detail : - _version = _version_base - elif '@' not in _version_base: - _version = '%s-%s' % (_version_base, _version_detail) + _, _, ret = sh_callout('cd %s && git rev-parse --git-dir && which git' + % root) + _in_git = (ret == 0) + + if not _in_git: + + with open(_version_path, 'w', encoding='utf-8') as fout: + fout.write(_version_base + '\n') + else: - _version = _version_base - - # make sure the version files exist for the runtime version inspection - _path = '%s/%s' % (root, _mod_root) - with open(_path + '/VERSION', 'w', encoding='utf-8') as fout: - fout.write(_version_base + '\n') - fout.write(_version + '\n') - - _sdist_name = '%s-%s.tar.gz' % (name, _version_base) - # _sdist_name = _sdist_name.replace('/', '-') - # _sdist_name = _sdist_name.replace('@', '-') - # _sdist_name = _sdist_name.replace('#', '-') - # _sdist_name = _sdist_name.replace('_', '-') - - # setuptools 69.5 does changes naming scheme - if not os.path.isfile('dist/%s' % _sdist_name): - _sdist_name = '%s-%s.tar.gz' % (name.replace('.', '_'), _version_base) - - if os.path.isfile('dist/%s' % _sdist_name): - # pip install stage 2 or easy_install stage 1 - # - # pip install will untar the sdist in a tmp tree. In that tmp - # tree, we won't be able to derive git version tags -- so we pack - # the formerly derived version as ./VERSION - shutil.move('VERSION', 'VERSION.bak') # backup - shutil.copy('%s/VERSION' % _path, 'VERSION') # version to use - os.system ('python3 setup.py sdist') # build sdist - shutil.copy('dist/%s' % _sdist_name, - '%s/%s' % (_mod_root, _sdist_name)) # copy into tree - shutil.move('VERSION.bak', 'VERSION') # restore version - - with open(_path + '/SDIST', 'w', encoding='utf-8') as fout: - fout.write(_sdist_name + '\n') - - return _version_base, _version_detail, _sdist_name, _path - except Exception as e: - raise RuntimeError('Could not extract/set version: %s' % e) from e + # get details from git + _out, _err, _ret = sh_callout('cd %s && git describe --tags --always' % root) + assert _ret == 0, 'git describe failed' + _out = _out.decode() + _out = _out.strip() + _version_tag = _out -# ------------------------------------------------------------------------------ -# get version info -- this will create VERSION and srcroot/VERSION -version, version_detail, sdist_name, path = get_version(mod_root) + _out, _err, _ret = sh_callout('cd %s && git branch --show-current' % root) + assert _ret == 0, 'git branch failed' + + _out = _out.decode() + _out = _out.strip() + + _version_branch = _out or 'detached' + _version_branch = _version_branch.replace('detached from ', '~') + + _version_short = _version_tag.split('-')[0] + _version_short = _version_short[1:] # strip the 'v' + + if _version_tag: + _version_detail = '%s-%s@%s' % (_version_base, _version_tag, + _version_branch) + else: + _version_detail = '%s@%s' % (_version_base, _version_branch) + + with open(_version_path, 'w', encoding='utf-8') as fout: + fout.write(_version_short + '\n') + fout.write(_version_base + '\n') + fout.write(_version_branch + '\n') + fout.write(_version_tag + '\n') + fout.write(_version_detail + '\n') + + return _version_base, _version_path + + except Exception as e: + _msg = 'Could not extract/set version: %s' % e + if _ret: + _msg += '\n' + _out + '\n\n' + _err + raise RuntimeError(_msg) from e # ------------------------------------------------------------------------------ -# check python version, should be >= 3.6 -if sys.hexversion < 0x03060000: - raise RuntimeError('ERROR: %s requires Python 3.6 or newer' % name) +# get version info -- this will create VERSION and srcroot/VERSION +version, version_path = get_version(mod_root) # ------------------------------------------------------------------------------ @@ -181,14 +131,8 @@ class RunTwine(Command): def initialize_options(self): pass def finalize_options(self): pass def run(self): - _, _, ret = sh_callout('python3 setup.py sdist upload -r pypi') - raise SystemExit(ret) - - -# ------------------------------------------------------------------------------ -# -df = list() -df.append(('share/%s/examples' % name, glob.glob('examples/*.py'))) + _, _, _ret = sh_callout('python3 setup.py sdist upload -r pypi') + raise SystemExit(_ret) # ------------------------------------------------------------------------------ @@ -201,11 +145,10 @@ def run(self): # setup_args = { 'name' : name, - # 'namespace_packages' : ['radical'], 'version' : version, - 'description' : 'A light-weight access layer for distributed ' - 'computing infrastructure ' - '(http://radical.rutgers.edu/)', + 'description' : descr, + 'long_description' : readme, + 'long_description_content_type' : 'text/markdown', 'author' : 'RADICAL Group at Rutgers University', 'author_email' : 'radical@rutgers.edu', 'maintainer' : 'The RADICAL Group', @@ -217,8 +160,8 @@ def run(self): 'Issues' : 'https://github.com/radical-cybertools/%s/issues' % name, }, 'license' : 'MIT', - 'keywords' : 'radical job saga', - 'python_requires' : '>=3.6', + 'keywords' : keywords, + 'python_requires' : '>=3.7', 'classifiers' : [ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', @@ -226,22 +169,22 @@ def run(self): 'License :: OSI Approved :: MIT License', 'Programming Language :: Python', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Topic :: Utilities', 'Topic :: System :: Distributed Computing', - 'Topic :: Scientific/Engineering :: Interface Engine/Protocol Translator', + 'Topic :: Scientific/Engineering', 'Operating System :: MacOS :: MacOS X', 'Operating System :: POSIX', 'Operating System :: Unix' ], 'packages' : find_namespace_packages('src', include=['radical.*']), 'package_dir' : {'': 'src'}, - 'scripts' : ['bin/radical-saga-version'], + 'scripts' : scripts, 'package_data' : {'': ['*.txt', '*.sh', '*.json', '*.gz', '*.c', - '*.md', 'VERSION', 'SDIST', sdist_name]}, + '*.md', 'VERSION']}, 'install_requires' : requirements, 'zip_safe' : False, - 'data_files' : df, + 'data_files' : data, 'cmdclass' : {'upload': RunTwine}, } @@ -253,11 +196,8 @@ def run(self): # ------------------------------------------------------------------------------ # clean temporary files from source tree -if sdist_level == 0: - os.system('rm -vrf src/%s.egg-info' % name) - os.system('rm -vf %s/%s' % (path, sdist_name)) - os.system('rm -vf %s/VERSION' % path) - os.system('rm -vf %s/SDIST' % path) +os.system('rm -vrf src/%s.egg-info' % name) +os.system('rm -vf %s' % version_path) # ------------------------------------------------------------------------------ From 739a2f9f7583e342b7ac26c40dd4d41b0bbe20e1 Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Wed, 8 May 2024 01:34:35 +0200 Subject: [PATCH 2/3] sync with RU --- src/radical/saga/__init__.py | 15 +++++++++++++-- src/radical/saga/version.py | 15 --------------- 2 files changed, 13 insertions(+), 17 deletions(-) delete mode 100644 src/radical/saga/version.py diff --git a/src/radical/saga/__init__.py b/src/radical/saga/__init__.py index 809ea12f2..aa39b4694 100644 --- a/src/radical/saga/__init__.py +++ b/src/radical/saga/__init__.py @@ -6,8 +6,6 @@ # ------------------------------------------------------------------------------ # - -from .version import * from .constants import * from .task import Task, Container @@ -39,5 +37,18 @@ from . import utils +# ------------------------------------------------------------------------------ +# +import os as _os +import radical.utils as _ru + +_mod_root = _os.path.dirname (__file__) + +version_short, version_base, version_branch, version_tag, version_detail \ + = _ru.get_version(_mod_root) +version = version_short +__version__ = version_detail + + # ------------------------------------------------------------------------------ diff --git a/src/radical/saga/version.py b/src/radical/saga/version.py deleted file mode 100644 index 8ec192c88..000000000 --- a/src/radical/saga/version.py +++ /dev/null @@ -1,15 +0,0 @@ - -import os as _os -import radical.utils as _ru - - -# ------------------------------------------------------------------------------ -# -_pwd = _os.path.dirname (__file__) -version_short, version_detail, version_base, version_branch, \ - sdist_name, sdist_path = _ru.get_version ([_pwd]) -version = version_short - - -# ------------------------------------------------------------------------------ - From 0d5f21c28e8a58e79e44126e8c8dc01122da94fa Mon Sep 17 00:00:00 2001 From: Andre Merzky Date: Wed, 8 May 2024 01:38:16 +0200 Subject: [PATCH 3/3] sync with RU --- src/radical/saga/__init__.py | 2 +- src/radical/saga/engine/engine.py | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/src/radical/saga/__init__.py b/src/radical/saga/__init__.py index aa39b4694..24cd78268 100644 --- a/src/radical/saga/__init__.py +++ b/src/radical/saga/__init__.py @@ -39,7 +39,7 @@ # ------------------------------------------------------------------------------ # -import os as _os +import os as _os import radical.utils as _ru _mod_root = _os.path.dirname (__file__) diff --git a/src/radical/saga/engine/engine.py b/src/radical/saga/engine/engine.py index ffd3ccc3c..aee248675 100644 --- a/src/radical/saga/engine/engine.py +++ b/src/radical/saga/engine/engine.py @@ -8,8 +8,7 @@ import radical.utils as ru -from .. import exceptions as rse -from ..version import * +from .. import exceptions as rse # ------------------------------------------------------------------------------ @@ -135,7 +134,6 @@ def __init__(self): # Initialize the logging, and log version (this is a singleton!) self._logger = ru.Logger('radical.saga') - self._logger.info('radical.saga version: %s' % version_detail) # load adaptors self._load_adaptors()