From 73966ee11ced89c1633c78cc31cd2b32dc81ad9e Mon Sep 17 00:00:00 2001 From: mak-454 Date: Tue, 29 Dec 2020 23:13:48 +0530 Subject: [PATCH 1/2] updated code to remove hardcodings of dkube URL and to access from the value set in env variable. --- sklearn/stock-pred-redshift/model/evaluation.py | 9 +++++++-- sklearn/stock-pred-redshift/model/printRecords.R | 5 +++-- sklearn/stock-pred-redshift/model/train.py | 8 ++++++-- 3 files changed, 16 insertions(+), 6 deletions(-) diff --git a/sklearn/stock-pred-redshift/model/evaluation.py b/sklearn/stock-pred-redshift/model/evaluation.py index f02fefaa..309e7609 100644 --- a/sklearn/stock-pred-redshift/model/evaluation.py +++ b/sklearn/stock-pred-redshift/model/evaluation.py @@ -12,6 +12,8 @@ r_database = os.getenv('DKUBE_DATASET_REDSHIFT_DATABASE', None) r_user = os.getenv('DKUBE_DATASET_REDSHIFT_USER', None) +dkube_url = os.getenv('DKUBE_CONTROLLER_NODEIP_ACCESS_URL', "http://dkube-controller-worker.dkube:5000") + if 'https:' in r_endpoint: p = '(?:https*://)?(?P[^:/ ]+).?(?P[0-9]*).*' m = re.search(p,r_endpoint) @@ -28,6 +30,9 @@ degree= int(sys.argv[5]) if len(sys.argv) > 5 else 2 def log_metrics(key, value): + #TODO - reporting has to be changed to mlflow api or expose dkube-exporter via gateway + if MLFLOW_METRIC_REPORTING != "True": + return url = "http://dkube-exporter.dkube:9401/mlflow-exporter" train_metrics = {} train_metrics['mode']="train" @@ -48,7 +53,7 @@ def eval_metrics(actual, pred): def rs_fetch_datasets(): user = os.getenv("DKUBE_USER_LOGIN_NAME") - url = "http://dkube-controller-worker.dkube:5000/dkube/v2/controller/users/%s/datums/class/dataset/datum/%s" + url = dkube_url + "/dkube/v2/controller/users/%s/datums/class/dataset/datum/%s" headers={"authorization": "Bearer "+os.getenv("DKUBE_USER_ACCESS_TOKEN")} datasets = [] for ds in json.load(open('/etc/dkube/redshift.json')): @@ -116,4 +121,4 @@ def get_data(filename): log_metrics('R2', r2) print('RMSE', rmse) print('R2', r2) - print('MAE', mae) \ No newline at end of file + print('MAE', mae) diff --git a/sklearn/stock-pred-redshift/model/printRecords.R b/sklearn/stock-pred-redshift/model/printRecords.R index ef8355cb..aec8857d 100644 --- a/sklearn/stock-pred-redshift/model/printRecords.R +++ b/sklearn/stock-pred-redshift/model/printRecords.R @@ -5,7 +5,8 @@ ds <- fromJSON("/etc/dkube/redshift.json") user <- Sys.getenv("LOGNAME") -url <- "http://dkube-controller-worker.dkube:5000/dkube/v2/controller/users/%s/datums/class/dataset/datum/%s" +dkube_url <- Sys.getenv("DKUBE_CONTROLLER_NODEIP_ACCESS_URL") +url <- sprintf("%s/dkube/v2/controller/users/%s/datums/class/dataset/datum/%s",dkube_url) token <- strsplit(Sys.getenv("RSTUDIO_HTTP_REFERER"), "=")[[1]][2] @@ -29,4 +30,4 @@ get_password <- function(user, db){ } } -get_password("dpaks", "dkube") \ No newline at end of file +get_password("dpaks", "dkube") diff --git a/sklearn/stock-pred-redshift/model/train.py b/sklearn/stock-pred-redshift/model/train.py index f3d77a94..5bbc686e 100644 --- a/sklearn/stock-pred-redshift/model/train.py +++ b/sklearn/stock-pred-redshift/model/train.py @@ -13,6 +13,7 @@ r_endpoint = os.getenv('DKUBE_DATASET_REDSHIFT_ENDPOINT', None) r_database = os.getenv('DKUBE_DATASET_REDSHIFT_DATABASE', None) r_user = os.getenv('DKUBE_DATASET_REDSHIFT_USER', None) +dkube_url = os.getenv('DKUBE_CONTROLLER_NODEIP_ACCESS_URL', "http://dkube-controller-worker.dkube:5000") if 'https:' in r_endpoint: @@ -39,7 +40,7 @@ def eval_metrics(actual, pred): def rs_fetch_datasets(): user = os.getenv("DKUBE_USER_LOGIN_NAME") - url = "http://dkube-controller-worker.dkube:5000/dkube/v2/controller/users/%s/datums/class/dataset/datum/%s" + url = dkube_url + "/dkube/v2/controller/users/%s/datums/class/dataset/datum/%s" headers={"authorization": "Bearer "+os.getenv("DKUBE_USER_ACCESS_TOKEN")} datasets = [] for ds in json.load(open('/etc/dkube/redshift.json')): @@ -58,6 +59,9 @@ def get_password(user, db): MODEL_DIR = '/opt/dkube/model' def log_metrics(key, value): + #TODO - reporting has to be changed to mlflow api or expose dkube-exporter via gateway + if MLFLOW_METRIC_REPORTING != "True": + return url = "http://dkube-exporter.dkube:9401/mlflow-exporter" train_metrics = {} train_metrics['mode']="train" @@ -145,4 +149,4 @@ def get_data(filename): img = cv2.imread('svm.png') log_histogram('Stock Prices', prices, step=1) - log_images('Stock Predictions Graph',[img]) \ No newline at end of file + log_images('Stock Predictions Graph',[img]) From 6508cdcda06bf32b093296b7e827e6ecca1b8c04 Mon Sep 17 00:00:00 2001 From: Md Shabbir Date: Thu, 21 Jan 2021 23:31:45 +0530 Subject: [PATCH 2/2] MLFLOW_METRIC_REPORTING env --- sklearn/stock-pred-redshift/model/train.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sklearn/stock-pred-redshift/model/train.py b/sklearn/stock-pred-redshift/model/train.py index 5bbc686e..b5d32f54 100644 --- a/sklearn/stock-pred-redshift/model/train.py +++ b/sklearn/stock-pred-redshift/model/train.py @@ -9,6 +9,7 @@ import joblib, psycopg2 import requests +MLFLOW_METRIC_REPORTING = os.getenv('MLFLOW_METRIC_REPORTING', "True") r_endpoint = os.getenv('DKUBE_DATASET_REDSHIFT_ENDPOINT', None) r_database = os.getenv('DKUBE_DATASET_REDSHIFT_DATABASE', None) @@ -44,7 +45,7 @@ def rs_fetch_datasets(): headers={"authorization": "Bearer "+os.getenv("DKUBE_USER_ACCESS_TOKEN")} datasets = [] for ds in json.load(open('/etc/dkube/redshift.json')): - resp = requests.get(url % (user, ds.get('rs_name')), headers=headers).json() + resp = requests.get(url % (user, ds.get('rs_name')), headers=headers, verify=False).json() ds['rs_password'] = resp['data']['datum']['redshift']['password'] datasets.append(ds) return datasets