slurm.py: Fix comment about missing state file

bart-registrant

@@ -60,7 +60,7 @@ DEFAULT_CONFIG_FILE = "/etc/bart/bart.conf"
 DEFAULT_LOGFILE = "/var/log/bart-registration.log"
 DEFAULT_HOSTKEY = "/etc/grid-security/hostkey.pem"
 DEFAULT_HOSTCERT = "/etc/grid-security/hostcert.pem"
-DEFAULT_CERTDIR = "/etc/grid-security/certificates"
+DEFAULT_CERTDIR = None
 DEFAULT_LOG_DIR = "/var/spool/bart/usagerecords/"
 DEFAULT_BATCH_SIZE = 100
 DEFAULT_TIMEOUT = "10,"
@@ -257,7 +257,6 @@ def httpRequest(url, method="GET", payload=None, ctxFactory=None, timeout=None):
     """
     params = {
         "timeout": timeout,
-        "verify": False,
     }
     if ctxFactory:
         params["cert"] = (ctxFactory.cert_path, ctxFactory.key_path)

bart/__init__.py

@@ -1,8 +0,0 @@
-# bart/__init__.py
-
-import time
-gmt = time.gmtime()
-
-# set this to "correct" version when making a release
-__version__ = 'svn-%04d%02d%02d' % (gmt.tm_year, gmt.tm_mon, gmt.tm_mday)
-

bart/config.py

@@ -22,7 +22,7 @@
 DEFAULT_LOG_FILE        = '/var/log/bart-logger.log'
 DEFAULT_LOG_DIR         = '/var/spool/bart/usagerecords'
 DEFAULT_STATEDIR        = '/var/spool/bart'
-DEFAULT_SUPPRESS_USERMAP_INFO = 'false'
+DEFAULT_SUPPRESS_USERMAP_INFO = 'true'
 DEFAULT_LOG_LEVEL       = 'INFO'
 DEFAULT_STDERR_LEVEL    = None
 

bart/slurm.py

@@ -30,7 +30,7 @@
 DEFAULT_IDTIMESTAMP = 'true'
 
 MAX_DAYS = 'max_days'
-MAX_DAYS_DEFAULT = 7
+MAX_DAYS_DEFAULT = 0
 
 # This fills in the "processors" field.
 PROCESSORS_UNIT         = 'processors_unit'
@@ -90,14 +90,17 @@ def versioncmp(a, b):
     aa = [ int(x) for x in re.findall(r"\d+", a) ]
     bb = [ int(x) for x in re.findall(r"\d+", b) ]
 
-    for i in range(min(len(aa), len(bb))):
+    a_length = len(aa)
+    b_length = len(bb)
+
+    for i in range(min(a_length, b_length)):
         if aa[i] < bb[i]:
             return -1
         elif aa[i] > bb[i]:
             return 1
 
     ## If we get here, all common components are equal. Decide by the number of components:
-    return cmp(a_length, b_length)
+    return (a_length > b_length) - (a_length < b_length)
 
 
 class SlurmBackend:
@@ -114,7 +117,7 @@ def __init__(self, state_starttime, max_days, user_list):
         while not self.results and croped:
             # Check if number of days since last run is > search_days, if so only
             # advance max_days days        
-            search_days += int(max_days)
+            search_days += max_days
             if max_days > 0 and datetime.datetime.now() - datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) > datetime.timedelta(days=search_days):
                 self.end_str = datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) + datetime.timedelta(days=search_days)
                 self.end_str = self.end_str.isoformat().split('.')[0]
@@ -318,7 +321,11 @@ def generateUsageRecords(self, hostname, user_map, project_map):
         """
         self.missing_user_mappings = {}
 
-        tlp = SlurmBackend(self.state, self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT), self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT))
+        max_days = int(self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT))
+        if max_days == 0:
+            max_days = (datetime.datetime.now() - datetime.datetime.strptime(self.state, "%Y-%m-%dT%H:%M:%S")).days + 1
+
+        tlp = SlurmBackend(self.state, max_days, self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT))
 
         count = 0
         while True:
@@ -345,7 +352,7 @@ def parseGeneratorState(self,state):
         This is returns the last jobid processed.
         """
         if state is None or len(state) == 0:
-            # no statefile -> we start from 50000 (DEFAULT_STATEFILE_DEFAULT) seconds / 5.7 days ago
+            # no statefile -> we start from STATEFILE_DEFAULT seconds ago
             sfd = int(self.cfg.getConfigValue(SECTION, STATEFILE_DEFAULT, DEFAULT_STATEFILE_DEFAULT))
             dt = datetime.datetime.now()-datetime.timedelta(seconds=sfd)
             state = dt.isoformat().split('.')[0]

bart/usagerecord/urparser.py

@@ -6,13 +6,14 @@
 Copyright: Nordic Data Grid Facility (2010)
 """
 
+import logging
 import time
 
-from twisted.python import log
-
 from bart.ext import isodate
 from bart.usagerecord import urelements as ur
 
+logger = logging.getLogger(__name__)
+
 
 # date constants
 ISO_TIME_FORMAT   = "%Y-%m-%dT%H:%M:%SZ" # if we want to convert back some time
@@ -26,23 +27,23 @@ def parseBoolean(value):
     elif value == '0' or value.lower() == 'false':
         return False
     else:
-        log.msg('Failed to parse value %s into boolean' % value, system='sgas.UsageRecord')
+        logger.info('Failed to parse value %s into boolean' % value)
         return None
 
 
 def parseInt(value):
     try:
         return int(value)
     except ValueError:
-        log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord')
+        logger.info("Failed to parse float: %s" % value)
         return None
 
 
 def parseFloat(value):
     try:
         return float(value)
     except ValueError:
-        log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord')
+        logger.info("Failed to parse float: %s" % value)
         return None
 
 
@@ -51,7 +52,7 @@ def parseISODuration(value):
         td = isodate.parse_duration(value)
         return (td.days * 3600*24) + td.seconds # screw microseconds
     except ValueError:
-        log.msg("Failed to parse duration: %s" % value, system='sgas.UsageRecord')
+        logger.info("Failed to parse duration: %s" % value)
         return None
 
 
@@ -60,10 +61,10 @@ def parseISODateTime(value):
         dt = isodate.parse_datetime(value)
         return time.strftime(JSON_DATETIME_FORMAT, dt.utctimetuple())
     except ValueError as e:
-        log.msg("Failed to parse datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord')
+        logger.info("Failed to parse datetime value: %s (%s)" % (value, str(e)))
         return None
     except isodate.ISO8601Error as e:
-        log.msg("Failed to parse ISO datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord')
+        logger.info("Failed to parse ISO datetime value: %s (%s)" % (value, str(e)))
         return None
 
 
@@ -139,8 +140,8 @@ def setIfNotNone(key, value):
 
         elif element.tag == ur.SUBMIT_TIME:         r['submit_time']    = parseISODateTime(element.text)
 
-        elif element.tag == ur.KSI2K_WALL_DURATION: log.msg('Got ksi2k wall duration element, ignoring (deprecated)', system='sgas.UsageRecord')
-        elif element.tag == ur.KSI2K_CPU_DURATION:  log.msg('Got ksi2k cpu duration element, ignoring (deprecated)', system='sgas.UsageRecord')
+        elif element.tag == ur.KSI2K_WALL_DURATION: logger.info('Got ksi2k wall duration element, ignoring (deprecated)')
+        elif element.tag == ur.KSI2K_CPU_DURATION:  logger.info('Got ksi2k cpu duration element, ignoring (deprecated)')
         elif element.tag == ur.USER_TIME:           r['user_time']           = parseISODuration(element.text)
         elif element.tag == ur.KERNEL_TIME:         r['kernel_time']         = parseISODuration(element.text)
         elif element.tag == ur.EXIT_CODE:           r['exit_code']           = parseInt(element.text)
@@ -188,7 +189,7 @@ def setIfNotNone(key, value):
                     r.setdefault('uploads', []).append(upload)
 
         else:
-            log.msg("Unhandled UR element: %s" % element.tag, system='sgas.UsageRecord')
+            logger.info("Unhandled UR element: %s" % element.tag)
 
     # backwards logger compatability
     # alot of loggers set node_count when they should have used processors, therefore:

bart/usagerecord/usagerecord.py

@@ -7,8 +7,8 @@
 # Author: Henrik Thostrup Jensen <[email protected]>
 # Copyright: Nordic Data Grid Facility (2009, 2010)
 
+import importlib.metadata
 import time
-from bart import __version__
 
 from bart.usagerecord import urelements as ur
 
@@ -25,7 +25,10 @@
 
 # values for the logger name + version
 LOGGER_NAME_VALUE    = 'SGAS-BaRT'
-LOGGER_VERSION_VALUE = __version__
+try:
+    LOGGER_VERSION_VALUE = importlib.metadata.version("sgas-bart")
+except importlib.metadata.PackageNotFoundError:
+    LOGGER_VERSION_VALUE = "unknown"
 
 # register namespaces in element tree so we get more readable xml files
 # the semantics of the xml files does not change due to this

bart/usagerecord/verify.py

@@ -6,19 +6,19 @@
 # Author: Magnus Jonsson <[email protected]>
 # Copyright: NeIC 2014
 
-
+import logging
 from bart.usagerecord import urparser
 
-from twisted.python import log
+logger = logging.getLogger(__name__)
 
 def verify(ur):
     try:
         d = urparser.xmlToDict(ur)
         if d['record_id'] is None:
-            log.err("No record_id found")
+            logger.error("No record_id found")
             return False
     except:
-        log.err("Failed to convert UR XML into dict")
+        logger.error("Failed to convert UR XML into dict")
         return False
     return True
 

docs/backend.slurm

@@ -20,8 +20,8 @@ idtimestamp: default=true
 Adds a timestamp to the recordid to be sure that recordid is unique if Slurm
 reuses the jobid.
 
-max_days: default=7
-Max number of days to process for every run of bart.
+max_days: default=0
+Max number of days to process for every run of bart, or 0 for number of days since last run.
 
 processors_unit: default=cpu
 Which element of the AllocTRES should be used as the PROCESSORS ("number of

pyproject.toml

@@ -0,0 +1,4 @@
+# Just enough pyproject.toml to allow building without distutils.
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"

setup.py

@@ -3,7 +3,10 @@
 from distutils.command.install import install
 from distutils.command.install_data import install_data
 
-from bart import __version__
+import time
+gmt = time.gmtime()
+
+version = '%04d%02d%02d' % (gmt.tm_year, gmt.tm_mon, gmt.tm_mday)
 
 
 # nasty global for relocation
@@ -49,14 +52,15 @@ def finalize_options(self):
 
 
 setup(name='sgas-bart',
-      version=__version__,
+      version=version,
       description='SGAS Batch system Reporting Tool',
       author='Henrik Thostrup Jensen / Magnus Jonsson',
       author_email='[email protected]',
       url='http://www.sgas.se/',
       packages=['bart','bart.usagerecord',
                     'bart.ext', 'bart.ext.isodate'],
       scripts = ['bart-logger', 'bart-registrant'],
+      install_requires = ['requests'],
       cmdclass = cmdclasses,
 
       data_files = [