diff --git a/bart-registrant b/bart-registrant index b8659e7..cf6e520 100755 --- a/bart-registrant +++ b/bart-registrant @@ -60,7 +60,7 @@ DEFAULT_CONFIG_FILE = "/etc/bart/bart.conf" DEFAULT_LOGFILE = "/var/log/bart-registration.log" DEFAULT_HOSTKEY = "/etc/grid-security/hostkey.pem" DEFAULT_HOSTCERT = "/etc/grid-security/hostcert.pem" -DEFAULT_CERTDIR = "/etc/grid-security/certificates" +DEFAULT_CERTDIR = None DEFAULT_LOG_DIR = "/var/spool/bart/usagerecords/" DEFAULT_BATCH_SIZE = 100 DEFAULT_TIMEOUT = "10," @@ -257,7 +257,6 @@ def httpRequest(url, method="GET", payload=None, ctxFactory=None, timeout=None): """ params = { "timeout": timeout, - "verify": False, } if ctxFactory: params["cert"] = (ctxFactory.cert_path, ctxFactory.key_path) diff --git a/bart/__init__.py b/bart/__init__.py index a9eee25..e69de29 100644 --- a/bart/__init__.py +++ b/bart/__init__.py @@ -1,8 +0,0 @@ -# bart/__init__.py - -import time -gmt = time.gmtime() - -# set this to "correct" version when making a release -__version__ = 'svn-%04d%02d%02d' % (gmt.tm_year, gmt.tm_mon, gmt.tm_mday) - diff --git a/bart/config.py b/bart/config.py index c263ad9..7ec197a 100644 --- a/bart/config.py +++ b/bart/config.py @@ -22,7 +22,7 @@ DEFAULT_LOG_FILE = '/var/log/bart-logger.log' DEFAULT_LOG_DIR = '/var/spool/bart/usagerecords' DEFAULT_STATEDIR = '/var/spool/bart' -DEFAULT_SUPPRESS_USERMAP_INFO = 'false' +DEFAULT_SUPPRESS_USERMAP_INFO = 'true' DEFAULT_LOG_LEVEL = 'INFO' DEFAULT_STDERR_LEVEL = None diff --git a/bart/slurm.py b/bart/slurm.py index 60e6880..2fef637 100644 --- a/bart/slurm.py +++ b/bart/slurm.py @@ -30,7 +30,7 @@ DEFAULT_IDTIMESTAMP = 'true' MAX_DAYS = 'max_days' -MAX_DAYS_DEFAULT = 7 +MAX_DAYS_DEFAULT = 0 # This fills in the "processors" field. PROCESSORS_UNIT = 'processors_unit' @@ -90,14 +90,17 @@ def versioncmp(a, b): aa = [ int(x) for x in re.findall(r"\d+", a) ] bb = [ int(x) for x in re.findall(r"\d+", b) ] - for i in range(min(len(aa), len(bb))): + a_length = len(aa) + b_length = len(bb) + + for i in range(min(a_length, b_length)): if aa[i] < bb[i]: return -1 elif aa[i] > bb[i]: return 1 ## If we get here, all common components are equal. Decide by the number of components: - return cmp(a_length, b_length) + return (a_length > b_length) - (a_length < b_length) class SlurmBackend: @@ -114,7 +117,7 @@ def __init__(self, state_starttime, max_days, user_list): while not self.results and croped: # Check if number of days since last run is > search_days, if so only # advance max_days days - search_days += int(max_days) + search_days += max_days if max_days > 0 and datetime.datetime.now() - datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) > datetime.timedelta(days=search_days): self.end_str = datetime.datetime.strptime( state_starttime, "%Y-%m-%dT%H:%M:%S" ) + datetime.timedelta(days=search_days) self.end_str = self.end_str.isoformat().split('.')[0] @@ -318,7 +321,11 @@ def generateUsageRecords(self, hostname, user_map, project_map): """ self.missing_user_mappings = {} - tlp = SlurmBackend(self.state, self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT), self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT)) + max_days = int(self.cfg.getConfigValue(SECTION, MAX_DAYS, MAX_DAYS_DEFAULT)) + if max_days == 0: + max_days = (datetime.datetime.now() - datetime.datetime.strptime(self.state, "%Y-%m-%dT%H:%M:%S")).days + 1 + + tlp = SlurmBackend(self.state, max_days, self.cfg.getConfigValue(SECTION, USERS, USERS_DEFAULT)) count = 0 while True: @@ -345,7 +352,7 @@ def parseGeneratorState(self,state): This is returns the last jobid processed. """ if state is None or len(state) == 0: - # no statefile -> we start from 50000 (DEFAULT_STATEFILE_DEFAULT) seconds / 5.7 days ago + # no statefile -> we start from STATEFILE_DEFAULT seconds ago sfd = int(self.cfg.getConfigValue(SECTION, STATEFILE_DEFAULT, DEFAULT_STATEFILE_DEFAULT)) dt = datetime.datetime.now()-datetime.timedelta(seconds=sfd) state = dt.isoformat().split('.')[0] diff --git a/bart/usagerecord/urparser.py b/bart/usagerecord/urparser.py index 71dd6d4..b9061ce 100644 --- a/bart/usagerecord/urparser.py +++ b/bart/usagerecord/urparser.py @@ -6,13 +6,14 @@ Copyright: Nordic Data Grid Facility (2010) """ +import logging import time -from twisted.python import log - from bart.ext import isodate from bart.usagerecord import urelements as ur +logger = logging.getLogger(__name__) + # date constants ISO_TIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" # if we want to convert back some time @@ -26,7 +27,7 @@ def parseBoolean(value): elif value == '0' or value.lower() == 'false': return False else: - log.msg('Failed to parse value %s into boolean' % value, system='sgas.UsageRecord') + logger.info('Failed to parse value %s into boolean' % value) return None @@ -34,7 +35,7 @@ def parseInt(value): try: return int(value) except ValueError: - log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord') + logger.info("Failed to parse float: %s" % value) return None @@ -42,7 +43,7 @@ def parseFloat(value): try: return float(value) except ValueError: - log.msg("Failed to parse float: %s" % value, system='sgas.UsageRecord') + logger.info("Failed to parse float: %s" % value) return None @@ -51,7 +52,7 @@ def parseISODuration(value): td = isodate.parse_duration(value) return (td.days * 3600*24) + td.seconds # screw microseconds except ValueError: - log.msg("Failed to parse duration: %s" % value, system='sgas.UsageRecord') + logger.info("Failed to parse duration: %s" % value) return None @@ -60,10 +61,10 @@ def parseISODateTime(value): dt = isodate.parse_datetime(value) return time.strftime(JSON_DATETIME_FORMAT, dt.utctimetuple()) except ValueError as e: - log.msg("Failed to parse datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord') + logger.info("Failed to parse datetime value: %s (%s)" % (value, str(e))) return None except isodate.ISO8601Error as e: - log.msg("Failed to parse ISO datetime value: %s (%s)" % (value, str(e)), system='sgas.UsageRecord') + logger.info("Failed to parse ISO datetime value: %s (%s)" % (value, str(e))) return None @@ -139,8 +140,8 @@ def setIfNotNone(key, value): elif element.tag == ur.SUBMIT_TIME: r['submit_time'] = parseISODateTime(element.text) - elif element.tag == ur.KSI2K_WALL_DURATION: log.msg('Got ksi2k wall duration element, ignoring (deprecated)', system='sgas.UsageRecord') - elif element.tag == ur.KSI2K_CPU_DURATION: log.msg('Got ksi2k cpu duration element, ignoring (deprecated)', system='sgas.UsageRecord') + elif element.tag == ur.KSI2K_WALL_DURATION: logger.info('Got ksi2k wall duration element, ignoring (deprecated)') + elif element.tag == ur.KSI2K_CPU_DURATION: logger.info('Got ksi2k cpu duration element, ignoring (deprecated)') elif element.tag == ur.USER_TIME: r['user_time'] = parseISODuration(element.text) elif element.tag == ur.KERNEL_TIME: r['kernel_time'] = parseISODuration(element.text) elif element.tag == ur.EXIT_CODE: r['exit_code'] = parseInt(element.text) @@ -188,7 +189,7 @@ def setIfNotNone(key, value): r.setdefault('uploads', []).append(upload) else: - log.msg("Unhandled UR element: %s" % element.tag, system='sgas.UsageRecord') + logger.info("Unhandled UR element: %s" % element.tag) # backwards logger compatability # alot of loggers set node_count when they should have used processors, therefore: diff --git a/bart/usagerecord/usagerecord.py b/bart/usagerecord/usagerecord.py index 67df05a..ce98331 100644 --- a/bart/usagerecord/usagerecord.py +++ b/bart/usagerecord/usagerecord.py @@ -7,8 +7,8 @@ # Author: Henrik Thostrup Jensen # Copyright: Nordic Data Grid Facility (2009, 2010) +import importlib.metadata import time -from bart import __version__ from bart.usagerecord import urelements as ur @@ -25,7 +25,10 @@ # values for the logger name + version LOGGER_NAME_VALUE = 'SGAS-BaRT' -LOGGER_VERSION_VALUE = __version__ +try: + LOGGER_VERSION_VALUE = importlib.metadata.version("sgas-bart") +except importlib.metadata.PackageNotFoundError: + LOGGER_VERSION_VALUE = "unknown" # register namespaces in element tree so we get more readable xml files # the semantics of the xml files does not change due to this diff --git a/bart/usagerecord/verify.py b/bart/usagerecord/verify.py index 361fbfe..9a2b212 100644 --- a/bart/usagerecord/verify.py +++ b/bart/usagerecord/verify.py @@ -6,19 +6,19 @@ # Author: Magnus Jonsson # Copyright: NeIC 2014 - +import logging from bart.usagerecord import urparser -from twisted.python import log +logger = logging.getLogger(__name__) def verify(ur): try: d = urparser.xmlToDict(ur) if d['record_id'] is None: - log.err("No record_id found") + logger.error("No record_id found") return False except: - log.err("Failed to convert UR XML into dict") + logger.error("Failed to convert UR XML into dict") return False return True diff --git a/docs/backend.slurm b/docs/backend.slurm index f7e403e..c8dabe2 100644 --- a/docs/backend.slurm +++ b/docs/backend.slurm @@ -20,8 +20,8 @@ idtimestamp: default=true Adds a timestamp to the recordid to be sure that recordid is unique if Slurm reuses the jobid. -max_days: default=7 -Max number of days to process for every run of bart. +max_days: default=0 +Max number of days to process for every run of bart, or 0 for number of days since last run. processors_unit: default=cpu Which element of the AllocTRES should be used as the PROCESSORS ("number of diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..36de028 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,4 @@ +# Just enough pyproject.toml to allow building without distutils. +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" diff --git a/setup.py b/setup.py index 6703802..7dbfd0f 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,10 @@ from distutils.command.install import install from distutils.command.install_data import install_data -from bart import __version__ +import time +gmt = time.gmtime() + +version = '%04d%02d%02d' % (gmt.tm_year, gmt.tm_mon, gmt.tm_mday) # nasty global for relocation @@ -49,7 +52,7 @@ def finalize_options(self): setup(name='sgas-bart', - version=__version__, + version=version, description='SGAS Batch system Reporting Tool', author='Henrik Thostrup Jensen / Magnus Jonsson', author_email='magnus@hpc2n.umu.se', @@ -57,6 +60,7 @@ def finalize_options(self): packages=['bart','bart.usagerecord', 'bart.ext', 'bart.ext.isodate'], scripts = ['bart-logger', 'bart-registrant'], + install_requires = ['requests'], cmdclass = cmdclasses, data_files = [