diff --git a/dkube/sdk/rsrcs/code.py b/dkube/sdk/rsrcs/code.py index a579d125..dcc3a1e4 100644 --- a/dkube/sdk/rsrcs/code.py +++ b/dkube/sdk/rsrcs/code.py @@ -27,7 +27,7 @@ class DkubeCode(object): """ - GIT_ACCESS_OPTS = ["apikey", "sshkey", "password"] + GIT_ACCESS_OPTS = ["none", "apikey", "sshkey", "password"] """ List of authentication options supported for git data source. diff --git a/dkube/sdk/rsrcs/dataset.py b/dkube/sdk/rsrcs/dataset.py index 97460c61..4daf2299 100644 --- a/dkube/sdk/rsrcs/dataset.py +++ b/dkube/sdk/rsrcs/dataset.py @@ -65,7 +65,7 @@ class DkubeDataset(object): """ - GIT_ACCESS_OPTS = ["apikey", "sshkey", "password"] + GIT_ACCESS_OPTS = ["none", "apikey", "sshkey", "password"] """ List of authentication options supported for git data source. @@ -93,7 +93,7 @@ def __init__(self, user, name=generate("dataset"), tags=None): access_key_id=None, access_key=None, bucket=None, prefix=None, endpoint=None) self.gitcreds = GitAccessCredentials( - username=None, password=None, apikey=None, sshkey=None, private=True) + username=None, password=None, apikey=None, sshkey=None, private=False) self.gitaccess = GitAccessInfo( path=None, url=None, branch=None, credentials=self.gitcreds) self.hostpath = DatumModelHostpath( @@ -159,10 +159,13 @@ def update_git_details(self, url, branch=None, authopt=GIT_ACCESS_OPTS[0], authv if authopt == 'apikey': self.gitcreds.apikey = authval + self.gitcreds.private = True elif authopt == 'password': self.gitcreds.password = authval + self.gitcreds.private = True elif authopt == 'sshkey': self.gitcreds.sshkey = authval + self.gitcreds.private = True def update_awss3_details(self, bucket, prefix, key, secret): """ diff --git a/dkube/sdk/rsrcs/model.py b/dkube/sdk/rsrcs/model.py index b7e92459..8933d0f1 100644 --- a/dkube/sdk/rsrcs/model.py +++ b/dkube/sdk/rsrcs/model.py @@ -60,7 +60,7 @@ class DkubeModel(object): :bash:`workstation` :- To upload data that is present on the local workstation. :bash:`Uploaded` """ - GIT_ACCESS_OPTS = ["apikey", "sshkey", "password"] + GIT_ACCESS_OPTS = ["none", "apikey", "sshkey", "password"] """ List of authentication options supported for git data source. @@ -87,7 +87,7 @@ def __init__(self, user, name=generate("dataset"), tags=None): access_key_id=None, access_key=None, bucket=None, prefix=None, endpoint=None) self.gitcreds = GitAccessCredentials( - username=None, password=None, apikey=None, sshkey=None, private=True) + username=None, password=None, apikey=None, sshkey=None, private=False) self.gitaccess = GitAccessInfo( path=None, url=None, branch=None, credentials=self.gitcreds) @@ -151,10 +151,13 @@ def update_git_details(self, url, branch=None, authopt=GIT_ACCESS_OPTS[0], authv if authopt == 'apikey': self.gitcreds.apikey = authval + self.gitcreds.private = True elif authopt == 'password': self.gitcreds.password = authval + self.gitcreds.private = True elif authopt == 'sshkey': self.gitcreds.sshkey = authval + self.gitcreds.private = True def update_awss3_details(self, bucket, prefix, key, secret): """ diff --git a/dkube/slurm/dkube-slurmjob-kfpl-op.yaml b/dkube/slurm/dkube-slurmjob-kfpl-op.yaml index 4f2feecd..8812fae4 100644 --- a/dkube/slurm/dkube-slurmjob-kfpl-op.yaml +++ b/dkube/slurm/dkube-slurmjob-kfpl-op.yaml @@ -32,21 +32,21 @@ implementation: user, token, run, url = 'https://dkube-proxy.dkube'): - import pprint import ast - import time - import os import json - import kfp - import json - import yaml - from pyfiglet import Figlet - from url_normalize import url_normalize + import os + import pprint + import time from collections import namedtuple from json import JSONDecodeError - from dkube.sdk.internal.dkube_api.models.job_model import JobModel + + import kfp + import yaml from dkube.sdk.internal.api_base import ApiBase + from dkube.sdk.internal.dkube_api.models.job_model import JobModel from dkube.slurm.job_properties import JobProperties + from pyfiglet import Figlet + from url_normalize import url_normalize if isinstance(run, JobModel) == True: run = run.to_dict() @@ -77,11 +77,35 @@ implementation: print("...................") run['parameters']['class'] = kind + + # check if datums are in right format user:datum + if 'datums' in run['parameters'][kind]: + datums = run['parameters'][kind]['datums'] + datasets = datums.get('datasets', []) + if datasets != None: + for idx, item in enumerate(datasets): + if item['name'] != None and ':' not in item['name']: + datasets[idx]['name'] = user + ':' + item['name'] + models = datums.get('models', []) + if models != None: + for idx, item in enumerate(models): + if item['name'] != None and ':' not in item['name']: + models[idx]['name'] = user + ':' + item['name'] + outputs = datums.get('outputs', []) + if outputs != None: + for idx, item in enumerate(outputs): + if item['name'] != None and ':' not in item['name']: + outputs[idx]['name'] = user + ':' + item['name'] + code = datums.get('workspace', None) + if code != None and code['data'] != None and code['data']['name'] != None: + if ':' not in code['data']['name']: + code['data']['name'] = user + ':' + code['data']['name'] + # check if am running as pipeline component if os.getenv('pipeline', 'false').lower() == 'true': wfid, runid = os.getenv("wfid"), os.getenv("runid") run['name'] = runid - run['parameters']['training']['tags'].extend( + run['parameters'][kind]['tags'].extend( ['owner=pipeline', 'workflowid=' + wfid, 'runid=' + runid]) if run['parameters']['generated'] is None: run['parameters']['generated'] = dict() @@ -115,6 +139,7 @@ implementation: status = run['parameters']['generated']['status'] state, reason = status['state'], status['reason'] if state.lower() in ['complete', 'failed', 'error']: + recorded = state print( "run {} - completed with state {} and reason {}".format(name, state, reason)) break @@ -125,21 +150,26 @@ implementation: recorded = state time.sleep(10) + if recorded.lower() in ['failed', 'error']: + exit(1) + rundetails = json.dumps(run) uuid = run['parameters']['generated']['uuid'] lineage = api.get_run_lineage(kind, user, uuid) outputs = lineage['outputs'] - artifacts = [ + + isremote = lambda output: 'version' in output and output['version'] != None + artifacts=[ {'datum': output['version']['datum_name'], 'class': output['version']['datum_type'], 'version': output['version']['uuid'], 'index': output['version']['index'] - } - for output in outputs + } + for output in filter(isremote, outputs) ] - artifacts = json.dumps(artifacts) + artifacts=json.dumps(artifacts) - output = namedtuple('Outputs', ['artifacts', 'run_details']) + output=namedtuple('Outputs', ['artifacts', 'run_details']) return output(artifacts, rundetails) def _serialize_str(str_value: str) -> str: