Skip to content

set private=False appropriately. #61

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
mak-454 wants to merge 1 commit into
base: main
Choose a base branch
from
Open

set private=False appropriately. #61

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion dkube/sdk/rsrcs/code.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ class DkubeCode(object):

"""

GIT_ACCESS_OPTS = ["apikey", "sshkey", "password"]
GIT_ACCESS_OPTS = ["none", "apikey", "sshkey", "password"]

"""
List of authentication options supported for git data source.
Expand Down
7 changes: 5 additions & 2 deletions dkube/sdk/rsrcs/dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,7 @@ class DkubeDataset(object):

"""

GIT_ACCESS_OPTS = ["apikey", "sshkey", "password"]
GIT_ACCESS_OPTS = ["none", "apikey", "sshkey", "password"]
"""
List of authentication options supported for git data source.

Expand Down Expand Up @@ -93,7 +93,7 @@ def __init__(self, user, name=generate("dataset"), tags=None):
access_key_id=None, access_key=None, bucket=None, prefix=None, endpoint=None)

self.gitcreds = GitAccessCredentials(
username=None, password=None, apikey=None, sshkey=None, private=True)
username=None, password=None, apikey=None, sshkey=None, private=False)
self.gitaccess = GitAccessInfo(
path=None, url=None, branch=None, credentials=self.gitcreds)
self.hostpath = DatumModelHostpath(
Expand Down Expand Up @@ -159,10 +159,13 @@ def update_git_details(self, url, branch=None, authopt=GIT_ACCESS_OPTS[0], authv

if authopt == 'apikey':
self.gitcreds.apikey = authval
self.gitcreds.private = True
elif authopt == 'password':
self.gitcreds.password = authval
self.gitcreds.private = True
elif authopt == 'sshkey':
self.gitcreds.sshkey = authval
self.gitcreds.private = True

def update_awss3_details(self, bucket, prefix, key, secret):
"""
Expand Down
7 changes: 5 additions & 2 deletions dkube/sdk/rsrcs/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ class DkubeModel(object):
:bash:`workstation` :- To upload data that is present on the local workstation. :bash:`Uploaded`
"""

GIT_ACCESS_OPTS = ["apikey", "sshkey", "password"]
GIT_ACCESS_OPTS = ["none", "apikey", "sshkey", "password"]

"""
List of authentication options supported for git data source.
Expand All @@ -87,7 +87,7 @@ def __init__(self, user, name=generate("dataset"), tags=None):
access_key_id=None, access_key=None, bucket=None, prefix=None, endpoint=None)

self.gitcreds = GitAccessCredentials(
username=None, password=None, apikey=None, sshkey=None, private=True)
username=None, password=None, apikey=None, sshkey=None, private=False)
self.gitaccess = GitAccessInfo(
path=None, url=None, branch=None, credentials=self.gitcreds)

Expand Down Expand Up @@ -151,10 +151,13 @@ def update_git_details(self, url, branch=None, authopt=GIT_ACCESS_OPTS[0], authv

if authopt == 'apikey':
self.gitcreds.apikey = authval
self.gitcreds.private = True
elif authopt == 'password':
self.gitcreds.password = authval
self.gitcreds.private = True
elif authopt == 'sshkey':
self.gitcreds.sshkey = authval
self.gitcreds.private = True

def update_awss3_details(self, bucket, prefix, key, secret):
"""
Expand Down
60 changes: 45 additions & 15 deletions dkube/slurm/dkube-slurmjob-kfpl-op.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,21 +32,21 @@ implementation:
user, token, run,
url = 'https://dkube-proxy.dkube'):

import pprint
import ast
import time
import os
import json
import kfp
import json
import yaml
from pyfiglet import Figlet
from url_normalize import url_normalize
import os
import pprint
import time
from collections import namedtuple
from json import JSONDecodeError
from dkube.sdk.internal.dkube_api.models.job_model import JobModel

import kfp
import yaml
from dkube.sdk.internal.api_base import ApiBase
from dkube.sdk.internal.dkube_api.models.job_model import JobModel
from dkube.slurm.job_properties import JobProperties
from pyfiglet import Figlet
from url_normalize import url_normalize

if isinstance(run, JobModel) == True:
run = run.to_dict()
Expand Down Expand Up @@ -77,11 +77,35 @@ implementation:
print("...................")

run['parameters']['class'] = kind

# check if datums are in right format user:datum
if 'datums' in run['parameters'][kind]:
datums = run['parameters'][kind]['datums']
datasets = datums.get('datasets', [])
if datasets != None:
for idx, item in enumerate(datasets):
if item['name'] != None and ':' not in item['name']:
datasets[idx]['name'] = user + ':' + item['name']
models = datums.get('models', [])
if models != None:
for idx, item in enumerate(models):
if item['name'] != None and ':' not in item['name']:
models[idx]['name'] = user + ':' + item['name']
outputs = datums.get('outputs', [])
if outputs != None:
for idx, item in enumerate(outputs):
if item['name'] != None and ':' not in item['name']:
outputs[idx]['name'] = user + ':' + item['name']
code = datums.get('workspace', None)
if code != None and code['data'] != None and code['data']['name'] != None:
if ':' not in code['data']['name']:
code['data']['name'] = user + ':' + code['data']['name']

# check if am running as pipeline component
if os.getenv('pipeline', 'false').lower() == 'true':
wfid, runid = os.getenv("wfid"), os.getenv("runid")
run['name'] = runid
run['parameters']['training']['tags'].extend(
run['parameters'][kind]['tags'].extend(
['owner=pipeline', 'workflowid=' + wfid, 'runid=' + runid])
if run['parameters']['generated'] is None:
run['parameters']['generated'] = dict()
Expand Down Expand Up @@ -115,6 +139,7 @@ implementation:
status = run['parameters']['generated']['status']
state, reason = status['state'], status['reason']
if state.lower() in ['complete', 'failed', 'error']:
recorded = state
print(
"run {} - completed with state {} and reason {}".format(name, state, reason))
break
Expand All @@ -125,21 +150,26 @@ implementation:
recorded = state
time.sleep(10)

if recorded.lower() in ['failed', 'error']:
exit(1)

rundetails = json.dumps(run)

uuid = run['parameters']['generated']['uuid']
lineage = api.get_run_lineage(kind, user, uuid)
outputs = lineage['outputs']
artifacts = [

isremote = lambda output: 'version' in output and output['version'] != None
artifacts=[
{'datum': output['version']['datum_name'], 'class': output['version']['datum_type'],
'version': output['version']['uuid'], 'index': output['version']['index']
}
for output in outputs
}
for output in filter(isremote, outputs)
]

artifacts = json.dumps(artifacts)
artifacts=json.dumps(artifacts)

output = namedtuple('Outputs', ['artifacts', 'run_details'])
output=namedtuple('Outputs', ['artifacts', 'run_details'])
return output(artifacts, rundetails)

def _serialize_str(str_value: str) -> str:
Expand Down