From 769bd1b1e3c0adf32d87033a92bee0a67094c4d5 Mon Sep 17 00:00:00 2001 From: spike Date: Sat, 29 Jan 2022 15:51:52 -0500 Subject: [PATCH 1/7] added a check to make sure server is alive before trying to download data --- getDataFRF.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/getDataFRF.py b/getDataFRF.py index eb7fe77..b3cb033 100644 --- a/getDataFRF.py +++ b/getDataFRF.py @@ -48,7 +48,7 @@ def gettime(allEpoch, epochStart, epochEnd, indexRef=0): finally: return idx -def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000,**kwargs): +def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000, **kwargs): """Function grabs the netCDF file interested. Responsible for drilling down to specific monthly file if applicable to speed things up. @@ -95,6 +95,8 @@ def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000 if callingClass == 'getDataTestBed': # overwrite pName if calling for model data pName = u'cmtb' + assert os.system("ping -c 1 " + THREDDSloc) == 0, f"Not able to see {THREDDSloc) + # now set URL for netCDF file call, if start is None and end is None: ncfileURL = urljoin('[FillMismatch]'+THREDDSloc, pName, dataLoc) @@ -245,7 +247,7 @@ def __init__(self, d1, d2, **kwargs): self._comp_time() assert type(self.d2) == DT.datetime, 'd1 need to be in python "Datetime" data types' assert type(self.d1) == DT.datetime, 'd2 need to be in python "Datetime" data types' - + def _comp_time(self): """Test if times are backwards.""" assert self.d2 >= self.d1, 'finish time: end needs to be after start time: start' From 0613e0eee88b2a873122d10097a401368126f6b3 Mon Sep 17 00:00:00 2001 From: spike Date: Sat, 29 Jan 2022 15:56:35 -0500 Subject: [PATCH 2/7] changed the logic to return a connection error, instead of a assertion error. This will allow for it to be handled downstream as needed, more effectively? : : --- getDataFRF.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/getDataFRF.py b/getDataFRF.py index b3cb033..3b5f81c 100644 --- a/getDataFRF.py +++ b/getDataFRF.py @@ -95,7 +95,8 @@ def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000 if callingClass == 'getDataTestBed': # overwrite pName if calling for model data pName = u'cmtb' - assert os.system("ping -c 1 " + THREDDSloc) == 0, f"Not able to see {THREDDSloc) + if os.system("ping -c 1 " + THREDDSloc) ~= 0: + return ConnectionError(f"Not able to see {THREDDSloc)) # now set URL for netCDF file call, if start is None and end is None: From 12750575c4de320eb637336e1bff496b0fe5d402 Mon Sep 17 00:00:00 2001 From: spike Date: Mon, 31 Jan 2022 17:18:47 -0500 Subject: [PATCH 3/7] implmented flexible file loading at the function level (after initilization --- getDataFRF.py | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/getDataFRF.py b/getDataFRF.py index 3b5f81c..502bd09 100644 --- a/getDataFRF.py +++ b/getDataFRF.py @@ -95,8 +95,8 @@ def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000 if callingClass == 'getDataTestBed': # overwrite pName if calling for model data pName = u'cmtb' - if os.system("ping -c 1 " + THREDDSloc) ~= 0: - return ConnectionError(f"Not able to see {THREDDSloc)) + if os.system("ping -c 1 " + THREDDSloc) != 0: + return ConnectionError(f"Not able to see {THREDDSloc}") # now set URL for netCDF file call, if start is None and end is None: @@ -877,7 +877,7 @@ def getGaugeWL(self, gaugenumber=5, roundto=1): 'name': str(self.ncfile.title), } return wlpacket - def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=False): + def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=False, **kwargs): """This function gets the bathymetric data from the server. Args: @@ -915,15 +915,24 @@ def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=F """ # do check here on profile numbers + fname = kwargs.get('fname', None) + # acceptableProfileNumbers = [None, ] - self.dataloc = 'geomorphology/elevationTransects/survey/surveyTransects.ncml' # location - # of the gridded surveys - dataReturns = getnc(dataLoc=self.dataloc, callingClass=self.callingClass, server=self.server, - dtRound=1 * 60, epoch1=self.epochd1, epoch2=self.epochd2) - if len(dataReturns) == 2: + if fname is None: + self.dataloc = 'geomorphology/elevationTransects/survey/surveyTransects.ncml' # location + # of the gridded surveys + dataReturns = getnc(dataLoc=self.dataloc, callingClass=self.callingClass, server=self.server, + dtRound=1 * 60, epoch1=self.epochd1, epoch2=self.epochd2) + else: # mimic the returns from the get nc function with a specific file + ncfile = nc.Dataset(fname) # open the pipe to the ncfile + dataReturns = (ncfile, sb.baseRound(ncfile['time'][:], 60)) + + if dataReturns is None: + return None + elif len(dataReturns) == 2: self.ncfile = dataReturns[0] self.allEpoch = dataReturns[1] - indexRef=0 + indexRef=[0] elif len(dataReturns) == 3: self.ncfile = dataReturns[0] self.allEpoch = dataReturns[1] From 66d240a34362c6fad63dcb3f9c88927b8a997929 Mon Sep 17 00:00:00 2001 From: spike Date: Mon, 31 Jan 2022 17:34:25 -0500 Subject: [PATCH 4/7] now working implmentation using local file. --- getDataFRF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getDataFRF.py b/getDataFRF.py index 502bd09..d7d7b59 100644 --- a/getDataFRF.py +++ b/getDataFRF.py @@ -932,7 +932,7 @@ def getBathyTransectFromNC(self, profilenumbers=None, method=1, forceReturnAll=F elif len(dataReturns) == 2: self.ncfile = dataReturns[0] self.allEpoch = dataReturns[1] - indexRef=[0] + indexRef=[0, len(self.ncfile['time'][:])] elif len(dataReturns) == 3: self.ncfile = dataReturns[0] self.allEpoch = dataReturns[1] From 670276e4564b420eaf4af1ba559bf93692113278 Mon Sep 17 00:00:00 2001 From: spike Date: Sun, 27 Feb 2022 11:48:16 -0500 Subject: [PATCH 5/7] added some logic for building local cache --- getDataFRF.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/getDataFRF.py b/getDataFRF.py index d7d7b59..0096a11 100644 --- a/getDataFRF.py +++ b/getDataFRF.py @@ -95,9 +95,9 @@ def getnc(dataLoc, callingClass, epoch1=0, epoch2=0, dtRound=60, cutrange=100000 if callingClass == 'getDataTestBed': # overwrite pName if calling for model data pName = u'cmtb' - if os.system("ping -c 1 " + THREDDSloc) != 0: - return ConnectionError(f"Not able to see {THREDDSloc}") - + # if os.system("ping -c 1 " + THREDDSloc) != 0: + # return ConnectionError(f"Not able to see {THREDDSloc}") + # # now set URL for netCDF file call, if start is None and end is None: ncfileURL = urljoin('[FillMismatch]'+THREDDSloc, pName, dataLoc) @@ -774,9 +774,11 @@ def getWL(self, collectionlength=6): self.WLdataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1, epochEnd=self.epochd2) - if np.size(self.WLdataindex) > 1: + if np.size(self.WLdataindex) >= 1 and (self.WLdataindex != None): self.WLtime = nc.num2date(self.allEpoch[self.WLdataindex], self.ncfile['time'].units, only_use_cftime_datetimes=False) + # for singular WLtime indices, might need to increas the dimension so it makes the below lists/arrays of + # size one, to keep downstream compatibility self.WLpacket = { 'name': str(self.ncfile.title), 'WL': self.ncfile['waterLevel'][self.WLdataindex], @@ -789,8 +791,8 @@ def getWL(self, collectionlength=6): } # this is faster to calculate myself, than pull from server self.WLpacket['residual'] = self.WLpacket['WL'] - self.WLpacket['predictedWL'] - elif self.WLdataindex is not None and np.size(self.WLdataindex) == 1: - raise BaseException('you have 1 WL point, can the above be a >= logic or does 1 cause problems') + # elif self.WLdataindex is not None and np.size(self.WLdataindex) == 1: + # raise BaseException('you have 1 WL point, can the above be a >= logic or does 1 cause problems') else: print('ERROR: there is no WATER level Data for this time period!!!') self.WLpacket = None From c82f9ec4a44664bd56547899df0ec6d8a570479b Mon Sep 17 00:00:00 2001 From: spike Date: Sun, 27 Feb 2022 11:53:51 -0500 Subject: [PATCH 6/7] added mean elevation and variance of the dictionary to getlidardem dict --- getDataFRF.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/getDataFRF.py b/getDataFRF.py index eb7fe77..cf04f15 100644 --- a/getDataFRF.py +++ b/getDataFRF.py @@ -2129,6 +2129,8 @@ def getLidarDEM(self, **kwargs): 'lat': self.ncfile['latitude'][ys, xs], 'lon': self.ncfile['longitude'][ys,xs] } + DEMdata['elevation_mean'] = np.nanmean(DEMdata['elevation'], axis=0) + DEMdata['elevation_var'] = np.nanvar(DEMdata['elevation'], axis=0) return DEMdata def getBathyRegionalDEM(self, utmEmin, utmEmax, utmNmin, utmNmax): From e5e347434fe50044489af1e321f4ea4abdb3f0b4 Mon Sep 17 00:00:00 2001 From: spike Date: Wed, 16 Nov 2022 22:09:36 -0500 Subject: [PATCH 7/7] updating != to is not --- getDataFRF.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/getDataFRF.py b/getDataFRF.py index 72a71ee..8e7ff1c 100644 --- a/getDataFRF.py +++ b/getDataFRF.py @@ -774,7 +774,7 @@ def getWL(self, collectionlength=6): self.WLdataindex = gettime(allEpoch=self.allEpoch, epochStart=self.epochd1, epochEnd=self.epochd2) - if np.size(self.WLdataindex) >= 1 and (self.WLdataindex != None): + if np.size(self.WLdataindex) >= 1 and (self.WLdataindex is not None): self.WLtime = nc.num2date(self.allEpoch[self.WLdataindex], self.ncfile['time'].units, only_use_cftime_datetimes=False) # for singular WLtime indices, might need to increas the dimension so it makes the below lists/arrays of