Fix compilation with Cython 0.29 (#371)

* fix compilation with cython 0.29.37

* do not import LOCAL_CLUSTER directly, access it via settings module

Author: tazend

pyslurm/__init__.py

@@ -7,10 +7,6 @@
 
 sys.setdlopenflags(sys.getdlopenflags() | os.RTLD_GLOBAL | os.RTLD_DEEPBIND)
 
-# Initialize slurm api
-from pyslurm.api import slurm_init, slurm_fini
-slurm_init()
-
 from .version import __version__
 
 from pyslurm import db
@@ -35,3 +31,7 @@
 
 # The old API in deprecated.pyx
 from pyslurm.deprecated import *
+
+# Initialize slurm api
+from pyslurm.api import slurm_init, slurm_fini
+slurm_init()

pyslurm/api.pyx

@@ -22,6 +22,8 @@
 # cython: c_string_type=unicode, c_string_encoding=default
 # cython: language_level=3
 
+from pyslurm import settings
+
 
 def slurm_init(config_path=None):
     """Initialize the Slurm API.
@@ -36,6 +38,7 @@ def slurm_init(config_path=None):
             None, so libslurm will automatically detect its config.
     """
     slurm.slurm_init(cstr.from_unicode(config_path))
+    settings.init()
 
 
 def slurm_fini():

pyslurm/core/job/job.pyx

@@ -34,7 +34,7 @@ from typing import Union
 from pyslurm.utils import cstr, ctime
 from pyslurm.utils.uint import *
 from pyslurm.core.job.util import *
-from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm import settings
 from pyslurm import xcollections
 from pyslurm.core.error import (
     RPCError,
@@ -236,7 +236,7 @@ cdef class Job:
         self.ptr.job_id = job_id
         self.passwd = {}
         self.groups = {}
-        cstr.fmalloc(&self.ptr.cluster, LOCAL_CLUSTER)
+        cstr.fmalloc(&self.ptr.cluster, settings.LOCAL_CLUSTER)
         self.steps = JobSteps()
         self.stats = JobStatistics()
         self.pids = {}

pyslurm/core/job/step.pyx

@@ -26,7 +26,7 @@ from typing import Union
 from pyslurm.utils import cstr, ctime
 from pyslurm.utils.uint import *
 from pyslurm.core.error import RPCError, verify_rpc
-from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm import settings
 from pyslurm import xcollections
 from pyslurm.utils.helpers import (
     signal_to_num,
@@ -159,7 +159,7 @@ cdef class JobStep:
         self.id = step_id
         self.stats = JobStepStatistics()
         self.pids = {}
-        cstr.fmalloc(&self.ptr.cluster, LOCAL_CLUSTER)
+        cstr.fmalloc(&self.ptr.cluster, settings.LOCAL_CLUSTER)
 
         # Initialize attributes, if any were provided
         for k, v in kwargs.items():

pyslurm/core/node.pyx

@@ -28,7 +28,7 @@ from pyslurm.utils import ctime
 from pyslurm.utils.uint import *
 from pyslurm.core.error import RPCError, verify_rpc
 from pyslurm.utils.ctime import timestamp_to_date, _raw_time
-from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm import settings
 from pyslurm import xcollections
 from pyslurm.utils.helpers import (
     uid_to_name,
@@ -222,7 +222,7 @@ cdef class Node:
     def __init__(self, name=None, **kwargs):
         self._alloc_impl()
         self.name = name
-        self.cluster = LOCAL_CLUSTER
+        self.cluster = settings.LOCAL_CLUSTER
         for k, v in kwargs.items():
             setattr(self, k, v)
 
@@ -270,7 +270,7 @@ cdef class Node:
         wrap._alloc_info()
         wrap.passwd = {}
         wrap.groups = {}
-        wrap.cluster = LOCAL_CLUSTER
+        wrap.cluster = settings.LOCAL_CLUSTER
         memcpy(wrap.info, in_ptr, sizeof(node_info_t))
         return wrap
 

pyslurm/core/partition.pxd

@@ -56,7 +56,6 @@ from pyslurm.utils cimport cstr
 from pyslurm.utils cimport ctime
 from pyslurm.utils.ctime cimport time_t
 from pyslurm.utils.uint cimport *
-from pyslurm.core cimport slurmctld
 from pyslurm.xcollections cimport MultiClusterMap
 
 
@@ -218,7 +217,7 @@ cdef class Partition:
     cdef:
         partition_info_t *ptr
         int power_save_enabled
-        slurmctld.Config slurm_conf
+        slurm_conf
 
     cdef readonly cluster
 

pyslurm/core/partition.pyx

@@ -30,7 +30,8 @@ from pyslurm.utils.uint import *
 from pyslurm.core.error import RPCError, verify_rpc
 from pyslurm.utils.ctime import timestamp_to_date, _raw_time
 from pyslurm.constants import UNLIMITED
-from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm import settings
+from pyslurm.core import slurmctld
 from pyslurm.core.slurmctld.config import _get_memory
 from pyslurm import xcollections
 from pyslurm.utils.helpers import (
@@ -78,7 +79,6 @@ cdef class Partitions(MultiClusterMap):
             Partitions partitions = Partitions()
             int flags = slurm.SHOW_ALL
             Partition partition
-            slurmctld.Config slurm_conf
             int power_save_enabled = 0
 
         verify_rpc(slurm_load_partitions(0, &partitions.info, flags))
@@ -168,7 +168,7 @@ cdef class Partition:
     def __init__(self, name=None, **kwargs):
         self._alloc_impl()
         self.name = name
-        self.cluster = LOCAL_CLUSTER
+        self.cluster = settings.LOCAL_CLUSTER
         for k, v in kwargs.items():
             setattr(self, k, v)
 
@@ -194,7 +194,7 @@ cdef class Partition:
     cdef Partition from_ptr(partition_info_t *in_ptr):
         cdef Partition wrap = Partition.__new__(Partition)
         wrap._alloc_impl()
-        wrap.cluster = LOCAL_CLUSTER
+        wrap.cluster = settings.LOCAL_CLUSTER
         memcpy(wrap.ptr, in_ptr, sizeof(partition_info_t))
         return wrap
 
@@ -801,7 +801,7 @@ def _preempt_mode_str_to_int(mode):
     return pmode
 
 
-def _preempt_mode_int_to_str(mode, slurmctld.Config slurm_conf):
+def _preempt_mode_int_to_str(mode, slurm_conf):
     if mode == slurm.NO_VAL16:
         return slurm_conf.preempt_mode if slurm_conf else None
     else:

pyslurm/core/slurmctld/__init__.pxd

@@ -1,4 +0,0 @@
-# cython: c_string_type=unicode, c_string_encoding=default
-# cython: language_level=3
-
-from .config cimport Config

pyslurm/db/assoc.pyx

@@ -29,7 +29,7 @@ from pyslurm.utils.helpers import (
 )
 from pyslurm.utils.uint import *
 from pyslurm.db.connection import _open_conn_or_error
-from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm import settings
 from pyslurm import xcollections
 
 
@@ -65,7 +65,7 @@ cdef class Associations(MultiClusterMap):
         # Fetch Assoc Data
         assoc_data = SlurmList.wrap(slurmdb_associations_get(
             conn.ptr, cond.ptr))
-        
+
         if assoc_data.is_null:
             raise RPCError(msg="Failed to get Association data from slurmdbd")
 
@@ -135,7 +135,7 @@ cdef class Associations(MultiClusterMap):
         else:
             # Autodetects the last slurm error
             raise RPCError()
-        
+
         if not db_connection:
             # Autocommit if no connection was explicitly specified.
             conn.commit()
@@ -185,7 +185,7 @@ cdef class Association:
     def __init__(self, **kwargs):
         self._alloc_impl()
         self.id = 0
-        self.cluster = LOCAL_CLUSTER
+        self.cluster = settings.LOCAL_CLUSTER
         for k, v in kwargs.items():
             setattr(self, k, v)
 

pyslurm/db/job.pyx

@@ -24,10 +24,8 @@
 
 from typing import Union, Any
 from pyslurm.core.error import RPCError, PyslurmError
-from pyslurm.core import slurmctld
-from typing import Any
 from pyslurm.utils.uint import *
-from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm import settings
 from pyslurm import xcollections
 from pyslurm.utils.ctime import (
     date_to_timestamp,
@@ -106,7 +104,7 @@ cdef class JobFilter:
         if not self.clusters:
             # This is a requirement for some other parameters to function
             # correctly, like self.nodelist
-            return [LOCAL_CLUSTER]
+            return [settings.LOCAL_CLUSTER]
         elif self.clusters == "all":
             return None
         else:
@@ -450,7 +448,7 @@ cdef class Job:
         self._alloc_impl()
         self.ptr.jobid = int(job_id)
         cstr.fmalloc(&self.ptr.cluster,
-                     LOCAL_CLUSTER if not cluster else cluster)
+                     settings.LOCAL_CLUSTER if not cluster else cluster)
         self.qos_data = QualitiesOfService()
         self.steps = JobSteps()
         self.stats = JobStatistics()
@@ -511,7 +509,7 @@ cdef class Job:
             >>> db_job = pyslurm.db.Job.load(10000, with_script=True)
             >>> print(db_job.script)
         """
-        cluster = LOCAL_CLUSTER if not cluster else cluster
+        cluster = settings.LOCAL_CLUSTER if not cluster else cluster
         jfilter = JobFilter(ids=[int(job_id)], clusters=[cluster],
                             with_script=with_script, with_env=with_env)
         job = Jobs.load(jfilter).get((cluster, int(job_id)))

pyslurm/settings.pyx

@@ -22,12 +22,18 @@
 # cython: c_string_type=unicode, c_string_encoding=default
 # cython: language_level=3
 
-from pyslurm.core import slurmctld
 from pyslurm cimport slurm
 from pyslurm.utils cimport cstr
 
 
-LOCAL_CLUSTER = cstr.to_unicode(slurm.slurm_conf.cluster_name)
-if not LOCAL_CLUSTER:
-    slurm_conf = slurmctld.Config.load()
-    LOCAL_CLUSTER = slurm_conf.cluster_name
+LOCAL_CLUSTER = "UNKNOWN"
+
+
+def init():
+    from pyslurm.core import slurmctld
+
+    global LOCAL_CLUSTER
+    LOCAL_CLUSTER = cstr.to_unicode(slurm.slurm_conf.cluster_name)
+    if not LOCAL_CLUSTER:
+        slurm_conf = slurmctld.Config.load()
+        LOCAL_CLUSTER = slurm_conf.cluster_name

pyslurm/xcollections.pxd

@@ -95,3 +95,4 @@ cdef class MultiClusterMap:
         _key_type
         _val_type
         _id_attr
+        _cluster

pyslurm/xcollections.pyx

@@ -23,7 +23,7 @@
 # cython: language_level=3
 """Custom Collection utilities"""
 
-from pyslurm.settings import LOCAL_CLUSTER
+from pyslurm import settings
 import json
 from typing import Union, Any
 
@@ -193,6 +193,7 @@ cdef class MultiClusterMap:
         self._key_type = key_type
         self._val_type = val_type
         self._id_attr = id_attr
+        self._cluster = settings.LOCAL_CLUSTER
         if init_data:
             self._init_data(data)
 
@@ -201,15 +202,15 @@ cdef class MultiClusterMap:
             for item in data:
                 if isinstance(item, self._key_type):
                     item = self._val_type(item)
-                    if LOCAL_CLUSTER not in self.data:
-                        self.data[LOCAL_CLUSTER] = {}
+                    if self._cluster not in self.data:
+                        self.data[self._cluster] = {}
 
-                self.data[LOCAL_CLUSTER].update({self._item_id(item): item})
+                self.data[self._cluster].update({self._item_id(item): item})
         elif isinstance(data, str):
             itemlist = data.split(",")
             items = {self._key_type(item):self._val_type(item)
                      for item in itemlist}
-            self.data[LOCAL_CLUSTER] = items
+            self.data[self._cluster] = items
         elif isinstance(data, dict):
             self.update(data)
         elif data is not None:
@@ -223,8 +224,8 @@ cdef class MultiClusterMap:
 
     def _get_cluster(self):
         cluster = None
-        if not self.data or LOCAL_CLUSTER in self.data:
-            cluster = LOCAL_CLUSTER
+        if not self.data or self._cluster in self.data:
+            cluster = self._cluster
         else:
             try:
                 cluster = next(iter(self.keys()))
@@ -259,7 +260,7 @@ cdef class MultiClusterMap:
                 try:
                     cluster = self._get_cluster()
                 except KeyError:
-                    cluster = LOCAL_CLUSTER
+                    cluster = self._cluster
 
                 if not cluster in self.data:
                     self.data[cluster] = {}
@@ -394,7 +395,7 @@ cdef class MultiClusterMap:
         according to `next(iter(self.keys()))` will be used.
 
         Examples:
-            Get a Job from the LOCAL_CLUSTER
+            Get a Job from the local Cluster.
 
             >>> job_id = 1
             >>> job = data.get(job_id)