From 738de16a2572128708abb00a2a03f79113896897 Mon Sep 17 00:00:00 2001 From: James Rising Date: Tue, 13 May 2025 22:45:00 -0400 Subject: [PATCH 1/4] add global farmer --- adaptation/covariates.py | 59 +++++++++++++++++++++++++++++++++++++- adaptation/curvegen.py | 7 +++-- interpret/calcspec.py | 2 +- interpret/container.py | 17 ++++++----- interpret/specification.py | 2 +- 5 files changed, 73 insertions(+), 14 deletions(-) diff --git a/adaptation/covariates.py b/adaptation/covariates.py index 6c692b19..534c6a5a 100644 --- a/adaptation/covariates.py +++ b/adaptation/covariates.py @@ -32,7 +32,7 @@ from openest.generate import fast_dataset from impactlab_tools.utils import files from .econmodel import * -from datastore import agecohorts, irvalues, irregions +from datastore import agecohorts, irvalues, irregions, population from climate.yearlyreader import RandomYearlyAccess from interpret import averages, configs @@ -1370,3 +1370,60 @@ def get_current(self, region): """ covars = self.source.get_current(region) return self.make_power(covars) + +class GlobalAggregatedCovariator(Covariator): + """Spatially average the covariates across all regions. + + Parameters + ---------- + source + maxbaseline + """ + def __init__(self, source, maxbaseline, config=None): + super(GlobalAggregatedCovariator, self).__init__(maxbaseline, config=config) + pop_baseline_withyear = population.population_baseline_data(maxbaseline, maxbaseline, []) + self.regions = pop_baseline_withyear.keys() + self.pop_baseline = [pop_baseline_withyear[region].values()[0] for region in self.regions] + self.source = source + self.year_of_cache = None + self.byregion_cache = None # {region: { key-local: value }} + self.global_cache = None # {key: value } + + def get_current(self, region): + """ + Parameters + ---------- + country : str + + Returns + ------- + dict + """ + if self.byregion_cache: + return {(key + '-local'): self.byregion_cache[region][key] for key in self.byregion_cache[region]} | self.global_cache + + self.byregion_cache = {region: self.source.get_current(region) for region in self.regions} + + self.global_cache = {} + for key in self.byregion_cache[self.regions[0]]: + regionvalues = [self.byregion_cache[region][key] for region in self.regions] + self.global_cache[key] = np.average(regionvalues, weights=self.pop_baseline) + + return get_current(region) + + def get_update(self, region, year, ds): + """ + Parameters + ---------- + region : str + year : int + ds : xarray.Dataset + """ + self.source.get_update(region, year, ds) + + if self.year_of_cache == year: + return # already done + + self.year_of_cache = year + self.byregion_cache = None + self.global_cache = None diff --git a/adaptation/curvegen.py b/adaptation/curvegen.py index 42b7635c..5c6c03bf 100644 --- a/adaptation/curvegen.py +++ b/adaptation/curvegen.py @@ -195,7 +195,10 @@ class FarmerCurveGenerator(DelayedCurveGenerator): """ def __init__(self, curvegen, covariator, farmer='full', save_curve=True, endbaseline=2015): super(FarmerCurveGenerator, self).__init__(curvegen) - self.covariator = covariator + if farmer == 'global': + self.covariator = covariates.GlobalAggregatedCovariator(covariator, endbaseline) + else: + self.covariator = covariator self.farmer = farmer self.save_curve = save_curve self.lincom_last_covariates = {} @@ -236,7 +239,7 @@ def get_next_curve(self, region, year, *args, **kwargs): if self.farmer == 'full': covariates = self.covariator.offer_update(region, year, kwargs['weather']) curve = self.curvegen.get_curve(region, year, covariates) - elif self.farmer == 'noadapt': + elif self.farmer in ['noadapt', 'global']: curve = self.last_curves[region] elif self.farmer == 'incadapt': covariates = self.covariator.offer_update(region, year, None) diff --git a/interpret/calcspec.py b/interpret/calcspec.py index d5f5fc39..dbc3ed37 100644 --- a/interpret/calcspec.py +++ b/interpret/calcspec.py @@ -31,7 +31,7 @@ def prepare_interp_raw(csvv, weatherbundle, economicmodel, qvals, farmer="full", weatherbundle : generate.weather.WeatherBundle economicmodel : adaptation.econmodel.SSPEconomicModel qvals : generate.pvalses.ConstantDictionary - farmer : {"full", "noadapt", "incadapt"}, optional + farmer : {"full", "noadapt", "incadapt", "global"}, optional Adaptation scheme to use. specconf : dict or None, optional This is the model containing 'specifications' and 'calculation' keys. diff --git a/interpret/container.py b/interpret/container.py index 04e2e7e9..bdb61920 100644 --- a/interpret/container.py +++ b/interpret/container.py @@ -130,12 +130,11 @@ def produce_csvv(basename, csvv, module, specconf, targetdir, weatherbundle, eco # Lock in the values pvals[basename].lock() - if check_doit(targetdir, basename + "-noadapt", suffix, config): - print("No adaptation") - calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer='noadapt', config=config, standard=False) - effectset.generate(targetdir, basename + "-noadapt" + suffix, weatherbundle, calculation, specconf['description'] + ", with no adaptation.", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv) - - if check_doit(targetdir, basename + "-incadapt", suffix, config): - print("Income-only adaptation") - calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer='incadapt', config=config, standard=False) - effectset.generate(targetdir, basename + "-incadapt" + suffix, weatherbundle, calculation, specconf['description'] + ", with interpolation and only environmental adaptation.", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv) + suffixes = {'noadapt': "with no adaptation", + 'incadapt': "with interpolation and only environmental adaptation", + 'global': "with no adaptation and global covariates"} + for farmer, explain in suffixes.items(): + if check_doit(targetdir, basename + "-" + farmer, suffix, config): + print("Limited adaptation: " + explain) + calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer=farmer, config=config, standard=False) + effectset.generate(targetdir, basename + "-" + farmer + suffix, weatherbundle, calculation, specconf['description'] + ", " + explain + ".", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv) diff --git a/interpret/specification.py b/interpret/specification.py index cd866cd6..1b10bc98 100644 --- a/interpret/specification.py +++ b/interpret/specification.py @@ -475,7 +475,7 @@ def prepare_interp_raw(csvv, weatherbundle, economicmodel, qvals, farmer='full', weatherbundle : generate.weather.DailyWeatherBundle economicmodel : adaptation.econmodel.SSPEconomicModel qvals : generate.pvalses.ConstantDictionary - farmer : {'full', 'noadapt', 'incadapt'}, optional + farmer : {'full', 'noadapt', 'incadapt', 'global'}, optional Type of farmer adaptation. specconf : dict, optional Specification configuration. From a8759d7d75874c38c4baa5b5ea5ed84ca70f8b28 Mon Sep 17 00:00:00 2001 From: James Rising Date: Mon, 19 May 2025 14:29:41 -0500 Subject: [PATCH 2/4] working version --- adaptation/covariates.py | 10 +++++----- adaptation/curvegen.py | 3 ++- datastore/population.py | 24 ++++++++++++++---------- generate/loadmodels.py | 6 ++++++ interpret/configs.py | 5 ++++- interpret/specification.py | 18 ++++++++++++++---- 6 files changed, 45 insertions(+), 21 deletions(-) diff --git a/adaptation/covariates.py b/adaptation/covariates.py index 534c6a5a..9b41b80b 100644 --- a/adaptation/covariates.py +++ b/adaptation/covariates.py @@ -1381,9 +1381,9 @@ class GlobalAggregatedCovariator(Covariator): """ def __init__(self, source, maxbaseline, config=None): super(GlobalAggregatedCovariator, self).__init__(maxbaseline, config=config) - pop_baseline_withyear = population.population_baseline_data(maxbaseline, maxbaseline, []) + pop_baseline_withyear = population.population_baseline_data(2000, maxbaseline, [], add_adm0=False) self.regions = pop_baseline_withyear.keys() - self.pop_baseline = [pop_baseline_withyear[region].values()[0] for region in self.regions] + self.pop_baseline = [np.mean(list(pop_baseline_withyear[region].values())) for region in self.regions] self.source = source self.year_of_cache = None self.byregion_cache = None # {region: { key-local: value }} @@ -1400,16 +1400,16 @@ def get_current(self, region): dict """ if self.byregion_cache: - return {(key + '-local'): self.byregion_cache[region][key] for key in self.byregion_cache[region]} | self.global_cache + return {**{(key + '-local'): self.byregion_cache[region][key] for key in self.byregion_cache[region]}, **self.global_cache} self.byregion_cache = {region: self.source.get_current(region) for region in self.regions} self.global_cache = {} - for key in self.byregion_cache[self.regions[0]]: + for key in self.byregion_cache[list(self.regions)[0]]: regionvalues = [self.byregion_cache[region][key] for region in self.regions] self.global_cache[key] = np.average(regionvalues, weights=self.pop_baseline) - return get_current(region) + return self.get_current(region) def get_update(self, region, year, ds): """ diff --git a/adaptation/curvegen.py b/adaptation/curvegen.py index 5c6c03bf..a177880b 100644 --- a/adaptation/curvegen.py +++ b/adaptation/curvegen.py @@ -34,6 +34,7 @@ from openest.generate.curvegen import * from openest.generate import checks, fast_dataset, formatting, smart_curve, formattools from openest.models.curve import FlatCurve +from .covariates import GlobalAggregatedCovariator region_curves = {} @@ -196,7 +197,7 @@ class FarmerCurveGenerator(DelayedCurveGenerator): def __init__(self, curvegen, covariator, farmer='full', save_curve=True, endbaseline=2015): super(FarmerCurveGenerator, self).__init__(curvegen) if farmer == 'global': - self.covariator = covariates.GlobalAggregatedCovariator(covariator, endbaseline) + self.covariator = GlobalAggregatedCovariator(covariator, endbaseline) else: self.covariator = covariator self.farmer = farmer diff --git a/datastore/population.py b/datastore/population.py index ccd0f3f9..489bce11 100644 --- a/datastore/population.py +++ b/datastore/population.py @@ -41,9 +41,9 @@ def each_future_population(model, scenario, dependencies): yield region, year, value -def population_baseline_data(year0, year1, dependencies): +def population_baseline_data(year0, year1, dependencies, add_adm0=True): global population_baseline_cache - if (year0, year1) in population_baseline_cache: + if add_adm0 and (year0, year1) in population_baseline_cache: return population_baseline_cache[year0, year1] baselinedata = {} # {region: {year: value}} @@ -66,14 +66,18 @@ def population_baseline_data(year0, year1, dependencies): if len(region) > 3 and region[3] == '.': if region[:3] not in adm0s: - adm0s[region[:3]] = 0 - adm0s[region[:3]] += value - - for adm0 in adm0s: - assert adm0 not in baselinedata - baselinedata[adm0] = adm0s[adm0] - - population_baseline_cache[year0, year1] = baselinedata + adm0s[region[:3]] = {} + if year not in adm0s[region[:3]]: + adm0s[region[:3]][year] = 0 + adm0s[region[:3]][year] += value + + if add_adm0: + for adm0 in adm0s: + assert adm0 not in baselinedata + baselinedata[adm0] = adm0s[adm0] + + if add_adm0: # Only save if we have ADM0 as well + population_baseline_cache[year0, year1] = baselinedata return baselinedata def extend_population_future(baselinedata, year0, year1, regions, model, scenario, dependencies): diff --git a/generate/loadmodels.py b/generate/loadmodels.py index af7afdc3..820234d9 100644 --- a/generate/loadmodels.py +++ b/generate/loadmodels.py @@ -16,10 +16,14 @@ def single(bundle_iterator): for econ_model, econ_scenario, economicmodel in covariates.iterate_econmodels(): allecons.append((econ_scenario, econ_model, economicmodel)) + assert allecons, "Cannot find economic models." + allclims = [] for clim_scenario, clim_model, weatherbundle in bundle_iterator: allclims.append((clim_scenario, clim_model, weatherbundle)) + assert allclims, "Cannot find climate models." + allexogenous = [] for econ_scenario, econ_model, economicmodel in allecons: for clim_scenario, clim_model, weatherbundle in allclims: @@ -35,6 +39,8 @@ def single(bundle_iterator): return clim_scenario, clim_model, weatherbundle, econ_scenario, econ_model, economicmodel + assert False, "Could not find an appropriate single model." + def random_order(bundle_iterator, config=None): if config is None: config = {} diff --git a/interpret/configs.py b/interpret/configs.py index b869e653..8c88f584 100644 --- a/interpret/configs.py +++ b/interpret/configs.py @@ -472,7 +472,10 @@ def items(self): copydict = dict(self.parent.items()) copydict.update(dict(self.child.items())) return copydict.items() - + + def check_usage(self): + return self.parent.check_usage() + self.child.check_usage() + class ConfigList(MutableSequence): """Wrapper on lists contained in configuration dictionaries to monitor key access. diff --git a/interpret/specification.py b/interpret/specification.py index 1b10bc98..ec335f02 100644 --- a/interpret/specification.py +++ b/interpret/specification.py @@ -8,7 +8,7 @@ precipitation. """ -import re +import re, copy from collections.abc import Mapping, Sequence from adaptation import csvvfile, curvegen, curvegen_known, curvegen_arbitrary, covariates, constraints, parallel_covariates, parallel_econmodel from generate import parallel_weather @@ -123,7 +123,7 @@ def create_covariator(specconf, weatherbundle, economicmodel, config=None, quiet Parameters ---------- - specconf : dict, optional + specconf : dict Specification configuration. weatherbundle : generate.weather.DailyWeatherBundle economicmodel : adaptation.econmodel.SSPEconomicModel @@ -136,6 +136,16 @@ def create_covariator(specconf, weatherbundle, economicmodel, config=None, quiet """ if config is None: config = {} + + if farmer == 'global': + # Need all regions in covariates for global + if 'filter-region' in config: + config = configs.shallow_copy(config) + del config['filter-region'] + if 'filter-region' in specconf: + specconf = configs.shallow_copy(specconf) + del specconf['filter-region'] + if parallel_weather.is_parallel(weatherbundle) and parallel_econmodel.is_parallel(economicmodel): return parallel_covariates.create_covariator(specconf, weatherbundle, economicmodel, farmer) if 'covariates' in specconf: @@ -164,7 +174,7 @@ def create_curvegen(csvv, covariator, regions, farmer='full', specconf=None, get Various parameters and curve descriptions from CSVV file. covariator : adaptation.covariates.Covariator or None regions : xarray.Dataset - farmer : {'full', 'noadapt', 'incadapt'}, optional + farmer : {'full', 'noadapt', 'incadapt', 'global'}, optional Type of farmer adaptation. specconf : dict, optional Specification configuration. @@ -506,7 +516,7 @@ def prepare_interp_raw(csvv, weatherbundle, economicmodel, qvals, farmer='full', ) depenunit = specconf['depenunit'] - + covariator = create_covariator(specconf, weatherbundle, economicmodel, config, farmer=farmer) # Subset to regions (i.e. hierids) to act on. From ebb0bf852c14358004eaa700512568b1271179d3 Mon Sep 17 00:00:00 2001 From: James Rising Date: Tue, 20 May 2025 09:30:23 -0700 Subject: [PATCH 3/4] better name handling --- interpret/container.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/interpret/container.py b/interpret/container.py index bdb61920..15aa63e6 100644 --- a/interpret/container.py +++ b/interpret/container.py @@ -38,6 +38,17 @@ def check_doit(targetdir, basename, suffix, config, deletebad=False): return False +def check_dofarmer(farmer, config, weatherbundle): + farmers = config.get('do_farmers', []) + if farmers == False: + farmers = [] + elif farmers == True: + farmers = ['noadapt', 'incadapt'] + elif farmers == 'always': + farmers = ['noadapt', 'incadapt', 'histclim-noadapt', 'histclim-incadapt'] + + return farmer in farmers or (weatherbundle.is_historical() and ('histclim-' + farmer) in farmers) + def get_modules(config): models = config['models'] for model in models: @@ -125,16 +136,17 @@ def produce_csvv(basename, csvv, module, specconf, targetdir, weatherbundle, eco if profile: return - - if config.get('do_farmers', False) and (not weatherbundle.is_historical() or config['do_farmers'] == 'always'): - # Lock in the values - pvals[basename].lock() - - suffixes = {'noadapt': "with no adaptation", - 'incadapt': "with interpolation and only environmental adaptation", - 'global': "with no adaptation and global covariates"} - for farmer, explain in suffixes.items(): - if check_doit(targetdir, basename + "-" + farmer, suffix, config): - print("Limited adaptation: " + explain) - calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer=farmer, config=config, standard=False) - effectset.generate(targetdir, basename + "-" + farmer + suffix, weatherbundle, calculation, specconf['description'] + ", " + explain + ".", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv) + + # Do farmers, if requested + suffixes = {'noadapt': "with no adaptation", + 'incadapt': "with interpolation and only environmental adaptation", + 'global': "with no adaptation and global covariates"} + + for farmer, explain in suffixes.items(): + if check_dofarmer(farmer, config, weatherbundle) and check_doit(targetdir, basename + "-" + farmer, suffix, config): + # Lock in the values + pvals[basename].lock() + + print("Limited adaptation: " + explain) + calculation, dependencies, baseline_get_predictors = caller.call_prepare_interp(csvv, module, weatherbundle, economicmodel, pvals[basename], specconf=specconf, farmer=farmer, config=config, standard=False) + effectset.generate(targetdir, basename + "-" + farmer + suffix, weatherbundle, calculation, specconf['description'] + ", " + explain + ".", dependencies + weatherbundle.dependencies + economicmodel.dependencies, config, push_callback=lambda reg, yr, app: push_callback(reg, yr, app, baseline_get_predictors, basename), deltamethod_vcv=deltamethod_vcv) From 8bcd1e128b1b43fd53acace9ef3689c53429bc3e Mon Sep 17 00:00:00 2001 From: James Rising Date: Tue, 20 May 2025 17:19:20 -0700 Subject: [PATCH 4/4] more options in do_farmers --- docs/generate.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/docs/generate.md b/docs/generate.md index 0d9ed23a..91179daa 100644 --- a/docs/generate.md +++ b/docs/generate.md @@ -34,10 +34,13 @@ included in the generate configuration file. a median or monte carlo run, but setting this to `false` will skip them. - - `do_farmers`: true, false, or 'always'; if true, alternative - assumptions of adaptation (income-only and no-adaptation) will be - generated. If 'always', alternative adaptation assumptions will be - calculated even with historical climate. + - `do_farmers`: true, false, 'always', or a list of 'noadapt', + 'incadapt', 'global', 'histclim-noadapt', 'histclim-incadapt', and + 'histclim-global'; if true, alternative assumptions of adaptation + (income-only and no-adaptation) will be generated. If 'always', + alternative adaptation assumptions will be calculated even with + historical climate. To generate arbitrary adaptation assumptions, + provide a list, e.g., ['noadapt', 'incadapt', 'global']. - `do_single`: true or false (default): Should we stop after a single target directory?