Sb/lots o changes

.cmtbSettings

@@ -0,0 +1,20 @@
+## This file is to hold system defaults so you don't have to include them in the input yaml
+# the input yaml should overwrite these values as needed
+#
+####################################
+#
+runFlag: true
+analyzeFlag: true
+plotFlag: false
+pbsFlag: false
+generateFlag: true
+namelistFlag: false
+updateBathy: true
+hpcSettings:
+   hpcCores: 36
+   hpcNodes: 1
+testName: ''
+logfileLoc: ./data/logs
+workingDirectory: ./data               # this is the directory where simulation files and QA/QC plots are made and existing architecture takes over
+netCDFdir: ./thredds_data             # this is the base directory where netCDF files are output
+TDSdatabase: ./TDSlocationGrabber/database  # location for TDS data base pickle (w/o extension)

.editorconfig

@@ -0,0 +1,15 @@
+[*]
+charset=utf-8
+end_of_line=lf
+insert_final_newline=false
+indent_style=space
+indent_size=4
+
+[*.json]
+indent_style=space
+indent_size=2
+
+[{*.yml,*.yaml}]
+indent_style=space
+indent_size=2
+

.gitignore

@@ -0,0 +1,62 @@
+# Data and netCDF4 files
+data/
+ncFiles/
+
+# Windows image file caches
+Thumbs.db
+ehthumbs.db
+
+# Folder config file
+Desktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows Installer files
+*.cab
+*.msi
+*.msm
+*.msp
+
+# Windows shortcuts
+*.lnk
+
+# =========================
+# Operating System Files
+# =========================
+
+# OSX
+# =========================
+
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+#pycharm fileS
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Python compile
+*.pyc
+.pyc

.gitmodules

@@ -1,9 +1,9 @@
-[submodule "prepdata"]
-	path = prepdata
-	url = [email protected]:erdc/prepdata.git
 [submodule "getdatatestbed"]
 	path = getdatatestbed
-	url = [email protected]:erdc/getdatatestbed.git
+	url = [email protected]:SBFRF/getdatatestbed.git
 [submodule "testbedutils"]
 	path = testbedutils
-	url = [email protected]:erdc/testbedutils.git
+	url = [email protected]:SBFRF/testbedutils.git
+[submodule "prepdata"]
+	path = prepdata
+	url = [email protected]:SBFRF/prepdata.git

ArchiveFolder/phaseResolved/FUNWAVES_play.py

@@ -0,0 +1,64 @@
+from getdatatestbed import getDataFRF
+import datetime as DT
+from prepdata import prepDataLib as pdl
+from testbedutils import sblib as sb
+import numpy as np
+## getting data for matt and gabby to write input files
+pd = pdl.PrepDataTools() # initalize for WL processing
+
+start = DT.datetime(2018, 10, 3, 17, 00)
+start1 = DT.datetime(2018, 10, 5, 17, 00)
+end = DT.datetime(2018, 10, 5, 18, 00)
+
+go1 = getDataFRF.getObs(start, start1, THREDDS='CHL')
+go2 = getDataFRF.getObs(start1, end, THREDDS='CHL')
+
+WL1 = go1.getWL()
+WL2 = go2.getWL()
+WL1 = pd.prep_WL(WL1, [DT.datetime(2018, 10, 4, 17)])
+WL2 = pd.prep_WL(WL2, [DT.datetime(2018, 10, 5, 17)])
+import netCDF4 as nc
+ncfile = nc.Dataset('https://chlthredds.erdc.dren.mil/thredds/dodsC/frf/geomorphology/elevationTransects/survey/FRF_20181004_1155_FRF_NAVD88_LARC_GPS_UTC_v20181012.nc')
+bathy = {'elevation': ncfile['elevation'][:],
+        'time': nc.num2date(ncfile['time'][:], 'seconds since 1970-01-01'),
+        'xFRF': ncfile['xFRF'][:],
+        'yFRF': ncfile['yFRF'][:],}
+# bathy1 = go1.getBathyTransectFromNC()
+wave1 = go1.getWaveSpec('8m-array')
+wave2 = go2.getWaveSpec('8m-array')
+
+# now merge to one dictionary
+def mergeDicts(dict1, dict2):
+
+    ds = [dict1, dict2]
+    d = {}
+    for k in dict1.keys():
+      d[k] = list(d[k] for d in ds)
+    return d
+WL = mergeDicts(WL1, WL2)
+
+idx1 = np.argwhere(wave1['time'] == DT.datetime(2018, 10, 4, 17)).squeeze()
+idx2 = np.argwhere(wave2['time'] == DT.datetime(2018, 10, 5, 17)).squeeze()
+wave1 = sb.reduceDict(wave1, idx1)
+waves = mergeDicts(wave1, wave2)
+import glob
+fname = glob.glob('/home/spike/Downloads/*.nc')
+toponcfile = nc.Dataset(fname[0])
+topo1 = {'elevation': toponcfile['elevation'][:],
+         'xFRF': toponcfile['xFRF'][:],
+         'yFRF': toponcfile['yFRF'][:]}
+toponcfile = nc.Dataset(fname[1])
+topo2 = {'elevation': toponcfile['elevation'][:],
+         'xFRF': toponcfile['xFRF'][:],
+         'yFRF': toponcfile['yFRF'][:]}
+topo = mergeDicts(topo1, topo2)
+import pickle
+fileOut = 'MattsFile.pickle'
+with open(fileOut, 'wb') as fid:
+    pickle.dump((waves, WL, bathy, topo), fid, protocol=pickle.HIGHEST_PROTOCOL)
+
+# to open
+
+with open(fileOut, 'rb') as fid:
+    data = pickle.load(fid)
+

ArchiveFolder/phaseResolved/funwavePlay.py

@@ -0,0 +1,86 @@
+import matplotlib
+# matplotlib.use('Agg')
+from testbedutils import waveLib as wl
+import numpy as np
+import datetime as DT
+from matplotlib import pyplot as plt
+from prepdata import inputOutput
+from plotting import operationalPlots as oP
+import os, glob, multiprocessing
+from testbedutils import sblib as sb
+import pickle
+###
+nSubSample = 5
+plotting = False
+loadData = False  # don't load in debug mode, takes forever!
+path_prefix = '/home/spike/cmtb/data/FUNWAVE'
+datestring = '20151005T000000Z'
+figureBaseFname = 'CMTB_waveModels_FUNWAVE_'
+bathyFile = '/home/spike/repos/cmtb/data/FUNWAVE/20151005T000000Z/depth.txt'
+pickleFname = os.path.join(path_prefix, figureBaseFname + 'TS.pickle')
+SeaSwellCutoff = 0.04
+WL = 1.4
+crossShoreTransectLine = 200
+bandAvg = 3
+########################################3
+# funwave Timeseries analysis
+
+fwio = inputOutput.funwaveIO(datestring, WL = WL)
+if loadData == True:
+    flist = sorted(glob.glob(os.path.join(path_prefix, datestring,'output/sta*')))
+    simData = fwio.loadFunwaveTransect(flist)
+    simData['time'] = np.array([DT.datetime.strptime(datestring, '%Y%m%dT%H%M%SZ') + DT.timedelta(seconds=i) for i in simData['time']])
+    simData['elevation'] = -fwio.loadBathy(bathyFile)[crossShoreTransectLine,]
+    with open(pickleFname, 'wb') as fid:
+        pickle.dump(simData, fid, protocol=pickle.HIGHEST_PROTOCOL)
+else:
+    with open(pickleFname, 'rb') as fid:
+        simData = pickle.load(fid)
+
+if plotting == True:
+    ## remove images before making them
+    imgList = glob.glob(os.path.join(path_prefix, datestring, 'figures', '*.png'))
+    [os.remove(ff) for ff in imgList]
+    ############### write a parallel data load function ##################
+    dataOut=[]
+    def parallel_generateCrossShoreTimeSeries(tidx):
+        ## generate a function that operates with only one input, can access local variable space
+        timeStep = simData['time'][tidx]
+        ofPlotName = os.path.join(path_prefix, datestring, 'figures', figureBaseFname + 'TS_' + timeStep.strftime('%Y%m%dT%H%M%S%fZ') +'.png')
+        oP.generate_CrossShoreTimeseries(ofPlotName, simData['eta'][tidx].squeeze(), -simData['elevation'], simData['xFRF'])
+        dataOut.append(ofPlotName)
+    ######################################################################
+    tstart = DT.datetime.now()
+    pool=multiprocessing.Pool(4)  # open multiprocessing pool
+    dataBack = pool.map(parallel_generateCrossShoreTimeSeries, range(0, len(simData['time']), nSubSample))
+    pool.close()
+    print('Took {} long to make all the plots in parallel {} processors'.format(DT.datetime.now() - tstart, 4))
+
+    # ## now make gif of waves moving across shore*
+    imgList = sorted(glob.glob(os.path.join(path_prefix, datestring, 'figures', figureBaseFname + '*TS_*.png')))
+    sb.makegif(imgList, os.path.join(path_prefix, datestring, 'figures', figureBaseFname + 'TS_{}.gif'.format(datestring)), dt=0.1)
+    print('Took {} long to make the movie and all the plots '.format(DT.datetime.now() - tstart))
+    # finish making images by taring files
+    tarOutFile  = os.path.join(path_prefix, datestring, 'figures', figureBaseFname + 'TS.tar.gz')
+    sb.myTarMaker(tarOutFile, imgList)
+    print('Took {} long to make all the plots, movie and tarball '.format(DT.datetime.now() - tstart))
+
+############### frequency analysis ##########################################################
+fspec, freqs = wl.timeSeriesAnalysis1D(simData['time'].squeeze(),simData['eta'].squeeze(), bandAvg=bandAvg, windowLength=30)
+total = wl.stats1D(fspec=fspec, frqbins=freqs, lowFreq=None, highFreq=None)
+SeaSwellStats = wl.stats1D(fspec=fspec, frqbins=freqs, lowFreq=SeaSwellCutoff, highFreq=None)
+IGstats = wl.stats1D(fspec=fspec, frqbins=freqs, lowFreq=None, highFreq=SeaSwellCutoff)
+HsTS = 4 * np.std(simData['eta'].squeeze(), axis=0)
+######################################################################################################
+####################################### Plot functions ######################################################
+#############################################################################################################
+setup = np.mean(simData['eta'], axis=0).squeeze()
+from plotting import operationalPlots as oP
+ofname = os.path.join(path_prefix, datestring, 'figures', figureBaseFname + '_crossShoreSummary.png')
+oP.plotCrossShoreSummaryTS(ofname, simData['xFRF'], simData['elevation'], total,
+                           SeaSwellStats, IGstats, setup=setup, WL=WL)
+ofname = os.path.join(path_prefix, datestring, 'figures', figureBaseFname + '_spectrograph.png')
+oP.crossShoreSpectrograph(ofname, simData['xFRF'], freqs, fspec)
+ofname = os.path.join(path_prefix, datestring, 'figures', figureBaseFname + '_surfaceTimeseries.png')
+oP.crossShoreSurfaceTS2D(ofname, simData['eta'], simData['xFRF'], simData['time'])
+