Skip to content

debug #2

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
YanCheng-Env wants to merge 7 commits into
base: master
Choose a base branch
from
Open

debug #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
cf13adc
debug
Mar 6, 2020
22289eb
debug
Mar 15, 2020
2ee4731
debug
Mar 15, 2020
a853528
debug merge images from the same satellite
Aug 17, 2020
eafe431
debug merge images from the same satellite
Aug 17, 2020
b3cf296
Merge branch 'master' of https://github.com/YanCheng-go/PlanetScope_t…
Aug 17, 2020
a9ef9e3
updates for processing online clipped files
Sep 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 40 additions & 23 deletions Utilities.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,9 @@ def __init__(self, gdal_osgeo_dir=default_gdal_osgeo_dir, work_dir=default_work_
self.records_path = None # File path of execution track document
self.id_list_download = None # a list of item id which will be downloaded

self.analytic_sr_name = [x for x in self.asset_types if 'analytic_sr' in x]
self.udm2_name = [x for x in self.asset_types if 'udm2' in x]

def shp_to_json(self):
'''
Convert AOI shapefile to json format that is required for retrieve imagery for specific location
Expand Down Expand Up @@ -218,7 +221,15 @@ def asset_attrs(asset_type):
'udm2': {
'suffix': 'udm2',
'data type': 'Byte'
}
},
'analytic_sr_clip': {
'suffix': 'AnalyticMS_SR_clip',
'data type': 'UInt16'
},
'udm2_clip': {
'suffix': 'udm2_clip',
'data type': 'Byte'
} # clipped ones need to be downloaded from the plant explorer manually
}
return switch.get(asset_type, 'None')

Expand Down Expand Up @@ -603,7 +614,7 @@ def udm2_setnull(self, file_list=None):
if file_list is None:
file_list = []
for i in self.id_list_download:
a = glob("{}\\{}*udm2.tif".format(input_dir, i))
a = glob("{}\\{}*{}.tif".format(input_dir, i, self.udm2_name[0]))
for j in a:
file_list.append(j)
else:
Expand All @@ -615,7 +626,7 @@ def udm2_setnull(self, file_list=None):
for file in file_list]))
exist_setnull = list(
set([file.split('\\')[-1].split('_{}_'.format(self.process_level))[0]
for file in file_list if 'setnull' in file]))
for file in file_list if 'setnull' in file])) # when the user defined file list does not contain _setnull file, this existing checking does not work...
new_setnull = [i for i in item_id_list if i not in exist_setnull]
if new_setnull:
if exist_setnull:
Expand Down Expand Up @@ -694,17 +705,23 @@ def merge(self, file_list=None):
date_list = [date for date in date_list if date not in date_list_exist]

for date in tqdm(date_list, total=len(date_list), unit="item", desc='Merging images'):
if asset_type == 'udm2':
if 'udm2' in asset_type:
file_list_new = glob("{}\\{}*{}_setnull.tif"
.format(input_dir, date, self.asset_attrs(asset_type)['suffix']))
else:
file_list_new = glob("{}\\{}*{}.tif".format(input_dir, date, self.asset_attrs(asset_type)['suffix']))
input_path = ' '.join(str(i) for i in file_list_new)
file_list_new = glob("{}\\{}*{}.tif".format(input_dir, date,
self.asset_attrs(asset_type)['suffix']))

satellite_id_list = list(set([x.split('\\')[-1].split('_{}_'.format(self.process_level))[0]
.split('_')[-1] for x in file_list_new]))

for satellite_id in satellite_id_list:
output_path = output_dir + '\\' + date + '_' + \
satellite_id + '_{}.tif'.format(self.asset_attrs(asset_type)['suffix'])
input_file_list = [file_path for file_path in file_list_new
if '_{}_{}_'.format(satellite_id, self.process_level) in file_path]
input_path = ' '.join(str(i) for i in input_file_list)
# print(input_path)
output_path = output_dir + '\\' + date + '_' + satellite_id + '_{}.tif'.format(
self.asset_attrs(asset_type)['suffix'])
self.gdal_merge(input_path, output_path, self.asset_attrs(asset_type)['data type'])

time_str = datetime.now().strftime("%Y%m%d-%H%M%S")
Expand Down Expand Up @@ -792,10 +809,10 @@ def clip(self, file_list=None):
for input_path in tqdm(file_list, total=len(file_list), unit="item", desc='Clipping images'):
output_name = input_path.split('\\')[-1]
output_path = output_dir + '\\' + output_name
if self.asset_attrs('udm2')['suffix'] in input_path:
data_type = self.asset_attrs('udm2')['data type']
if self.asset_attrs('analytic_sr')['suffix'] in input_path:
data_type = self.asset_attrs('analytic_sr')['data type']
if self.asset_attrs(self.udm2_name[0])['suffix'] in input_path:
data_type = self.asset_attrs(self.udm2_name[0])['data type']
if self.asset_attrs(self.analytic_sr_name[0])['suffix'] in input_path:
data_type = self.asset_attrs(self.analytic_sr_name[0])['data type']
self.gdal_clip(input_path, self.pixel_res(self.satellite), self.aoi_shp, output_path, data_type)

time_str = datetime.now().strftime("%Y%m%d-%H%M%S")
Expand Down Expand Up @@ -845,16 +862,16 @@ def band_algebra(self, output_type, file_list=None):
input_dir = self.work_dir + '\\' + self.output_dirs['clip']

if output_type == 'clear prob':
asset_type = 'udm2'
asset_type = self.udm2_name[0]
clear_prob_dir = self.work_dir + '\\' + self.output_dirs['clear prob']
if file_list is None:
file_list = []
for i in self.id_list_download:
a = glob("{}\\{}*{}.tif".format(input_dir, i, self.asset_attrs('udm2')['suffix']))
a = glob("{}\\{}*{}.tif".format(input_dir, i, self.asset_attrs(asset_type)['suffix']))
for j in a:
file_list.append(j)
else:
file_list = [file for file in file_list if self.asset_attrs('udm2')['suffix'] in file]
file_list = [file for file in file_list if self.asset_attrs(asset_type)['suffix'] in file]

# Check existing clipped images and remove the latest file, in case it is not complete
file_list_exist = glob('{}\\*.tif'.format(clear_prob_dir))
Expand All @@ -865,11 +882,11 @@ def band_algebra(self, output_type, file_list=None):
latest_file = max(file_list_exist, key=os.path.getctime)
os.remove(latest_file)
file_list_exist.remove(latest_file)
file_list = [file for file in file_list if file.split('\\')[-1].split(
file_list = [input_dir + file for file in file_list if file.split('\\')[-1].split(
'_{}'.format(self.asset_attrs(asset_type)['suffix']))[0] not in item_id_list_exist]

for udm2_path in file_list:
clear_prob_path = clear_prob_dir + '\\' + udm2_path.split('\\')[-1].split('_{}'.format(self.asset_attrs('udm2')['suffix']))[0] + '_clearprob.tif'
clear_prob_path = clear_prob_dir + '\\' + udm2_path.split('\\')[-1].split('_{}'.format(self.asset_attrs(asset_type)['suffix']))[0] + '_clearprob.tif'
self.gdal_calc_clear_prob(input_path=udm2_path, output_path=clear_prob_path)
print('Finish GDAL Calculation :)')
print('The outputs have been saved in this directory: ' + clear_prob_dir)
Expand All @@ -879,14 +896,14 @@ def band_algebra(self, output_type, file_list=None):
if file_list is None:
file_list = []
for i in self.id_list_download:
a = glob("{}\\{}*SR.tif".format(input_dir, i))
a = glob("{}\\{}*_{}.tif".format(input_dir, i, self.asset_attrs(self.analytic_sr_name[0])['suffix']))
for j in a:
file_list.append(j)
else:
file_list = [file for file in file_list if 'SR' in file]
file_list = [file for file in file_list if self.asset_attrs(self.analytic_sr_name[0])['suffix'] in file]
for sr_path in file_list:
ndvi_path = ndvi_dir + '\\' + sr_path.split('\\')[-1].split(
'_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0] + '_ndvi.tif'
'_{}'.format(self.asset_attrs(self.analytic_sr_name[0])['suffix']))[0] + '_ndvi.tif'
self.gdal_calc_ndvi(input_path=sr_path, output_path=ndvi_path)
time_str = datetime.now().strftime("%Y%m%d-%H%M%S")
print('Finish GDAL Calculation :)')
Expand Down Expand Up @@ -949,21 +966,21 @@ def clip_clear_perc(self, shapefile_path, clear_perc_min, save_rgb=True, save_cl
shp_name = shapefile_path.split('\\')[-1].split('.shp')[0]
# List merged and clipped udm2 file path
if file_list is None:
file_list = glob('{}\\{}\\*udm2.tif'.format(self.work_dir, self.output_dirs['clip']))
file_list = glob('{}\\{}\\*_{}.tif'.format(self.work_dir, self.output_dirs['clip'], self.udm2_name[0]))
else:
file_list = [file for file in file_list if 'udm2' in file]

# Check existing file
# if save_clip is True:
# file_list_exist = glob('{}\\*.tif'.format(output_dir))
# if file_list_exist:
# item_id_list_exist = [file.split('\\')[0].split('_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0]
# item_id_list_exist = [file.split('\\')[-1].split('_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0]
# for file in file_list_exist]
# if self.remove_latest is True:
# latest_file = max(file_list_exist, key=os.path.getctime)
# os.remove(latest_file)
# file_list_exist.remove(latest_file)
# file_list = [file for file in file_list if file.split('\\')[0].split(
# file_list = [input_dir + file for file in file_list if file.split('\\')[-1].split(
# '_{}'.format(self.asset_attrs('analytic_sr')['suffix']))[0] not in item_id_list_exist]

asset_id_list = []
Expand Down
23 changes: 11 additions & 12 deletions main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,26 +17,25 @@
# You can also change the variables namely default_XXXXXX in the Utilities.py file
# In this case, you do not need to set these variables when you call functions in Utilities.py
# Set environment
ut.gdal_osgeo_dir = r'C:\Users\ChengY\AppData\Roaming\Python\Python37\site-packages\osgeo'
ut.gdal_osgeo_dir = r'C:\Python37\Anaconda3\Lib\site-packages\osgeo'

# Set work directory
ut.work_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD'
# Set folders for saving different outputs
ut.output_dirs = {'raw': 'raw', 'clip': 'clip', 'clipped_raw': 'clipped_raw', 'merge': 'merge',
ut.output_dirs = {'raw': 'raw_clip_test', 'clip': 'clip', 'clipped_raw': 'clipped_raw', 'merge': 'merge_clip_test',
'clear prob': 'clear_prob', 'NDVI': 'NDVI', 'clip clear perc': 'bomas'}
ut.api_key = "9cada8bc134546fe9c1b8bce5b71860f"
ut.api_key = "e912e7e40d4d4c92bd6cbdffef89b76c" #"9cada8bc134546fe9c1b8bce5b71860f"
ut.satellite = 'PS'
ut.proj_code = 32737
ut.dpi = 90

# Filter settings
ut.filter_items = ['date', 'cloud_cover', 'aoi']
ut.item_types = ["PSScene4Band"]
ut.process_level = '3B'
ut.asset_types = ['analytic_sr', 'udm2']
ut.asset_types = ['analytic_sr_clip', 'udm2_clip'] # download() cannot be used for on-the-fly clipped assets (with asset_type named as ['analytic_sr_clip', 'udm2_clip'])
# Set filter
ut.start_date = '2020-02-05'
ut.end_date = '2020-02-11'
ut.end_date = '2020-02-6'
ut.cloud_cover = 1
ut.aoi_shp = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\shp\Kapiti\Kapiti_Jun18_v2_prj.shp'
# Settings for raster visualization
Expand Down Expand Up @@ -74,12 +73,12 @@
#
#
# # =================================== Merge ======================================#
# # Set input directory that includes all data to be merged
# # the default directory is [..\raw], which is the automatically created folder for saving all downloaded images
# input_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\raw'
# file_list = glob("{}\\*udm2.tif".format(input_dir)) # only for udm2
# # file_list = glob("{}\\*.tif".format(input_dir)) # for all tif
# ut.merge(file_list=file_list)
# Set input directory that includes all data to be merged
# the default directory is [..\raw], which is the automatically created folder for saving all downloaded images
input_dir = r'C:\Users\ChengY\PycharmProjects\PyPlanetScope_WD\raw_online'
file_list = glob("{}\\*udm2_clip*.tif".format(input_dir)) #### The * after udm2_clip can avoid repeating set_null process if it has been down
# only for udm2_clip # for checking existing setnull file, better to write as "udm2_clip*" instead of udm2_clip... I mean include set_null file as well
ut.merge(file_list=file_list)
#
#
# # =================================== Clip ======================================#
Expand Down