From 6851311bbe9c1320447b7d58568b86fbc4803dc5 Mon Sep 17 00:00:00 2001 From: Opoku-Agyemang Date: Mon, 15 Jul 2024 14:53:31 -0700 Subject: [PATCH] refactor prints to log & ensure local|test mode don't send logs to sentry and cw --- src/_main_/celery/app.py | 4 +- src/_main_/settings.py | 23 +++---- src/_main_/utils/GeoIP.py | 5 +- src/_main_/utils/common.py | 2 - .../utils/massenergize_logger/__init__.py | 52 +++++++-------- src/_main_/utils/metrics/__init__.py | 2 +- src/_main_/utils/stage/__init__.py | 10 +-- src/_main_/utils/stage/logging.py | 50 ++++++++++++-- src/api/handlers/misc.py | 1 - src/api/services/auth.py | 5 +- src/api/services/event.py | 1 - src/api/services/userprofile.py | 3 +- src/api/store/community.py | 66 +++++++++---------- src/api/store/download.py | 1 - src/api/store/event.py | 2 +- src/api/store/graph.py | 2 +- src/api/store/media_library.py | 1 - src/api/store/misc.py | 41 +++++------- src/api/store/team.py | 1 - src/api/store/userprofile.py | 11 ++-- src/api/utils/api_utils.py | 6 +- .../create_campaign_template.py | 28 ++++---- src/carbon_calculator/CCDefaults.py | 4 +- src/carbon_calculator/carbonCalculator.py | 11 ++-- src/database/models.py | 5 +- .../contents_spacing_correction.py | 1 - .../database_tasks/update_actions_content.py | 3 +- src/task_queue/nudges/cadmin_events_nudge.py | 10 +-- .../nudges/postmark_sender_signature.py | 7 +- src/task_queue/nudges/user_event_nudge.py | 21 +++--- src/task_queue/views.py | 14 ++-- src/website/views.py | 2 +- 32 files changed, 206 insertions(+), 189 deletions(-) diff --git a/src/_main_/celery/app.py b/src/_main_/celery/app.py index 25e924f0f..03ac50824 100644 --- a/src/_main_/celery/app.py +++ b/src/_main_/celery/app.py @@ -3,7 +3,7 @@ from celery import Celery from celery import shared_task from _main_.celery.config import CeleryConfig -from celery.schedules import crontab +from _main_.utils.massenergize_logger import log os.environ.setdefault("DJANGO_SETTINGS_MODULE", "_main_.settings") app = Celery('massenergize_celeryapp') @@ -23,4 +23,4 @@ @shared_task(bind=True) def debug_task(self): - print('Request: {0!r}'.format(self.request)) + log.info('Request: {0!r}'.format(self.request)) diff --git a/src/_main_/settings.py b/src/_main_/settings.py index 6178eee7e..9d85e05c7 100644 --- a/src/_main_/settings.py +++ b/src/_main_/settings.py @@ -234,17 +234,18 @@ ### End Logger settings ### # Sentry Logging Initialization -sentry_dsn = os.environ.get('SENTRY_DSN') -if sentry_dsn: - sentry_sdk.init( - dsn=sentry_dsn, - integrations=[DjangoIntegration( - transaction_style='url', - middleware_spans=True, - ), - CeleryIntegration(), - ], - traces_sample_rate=1.0, +if EnvConfig.can_send_logs_to_external_watchers(): + sentry_dsn = os.environ.get('SENTRY_DSN') + if sentry_dsn: + sentry_sdk.init( + dsn=sentry_dsn, + integrations=[DjangoIntegration( + transaction_style='url', + middleware_spans=True, + ), + CeleryIntegration(), + ], + traces_sample_rate=1.0, # If you wish to associate users to errors (assuming you are using diff --git a/src/_main_/utils/GeoIP.py b/src/_main_/utils/GeoIP.py index 29b9fe38a..3f870cd46 100644 --- a/src/_main_/utils/GeoIP.py +++ b/src/_main_/utils/GeoIP.py @@ -35,7 +35,6 @@ def getBrowser(self, request): # iPhone's user agent string #ua_string = 'Mozilla/5.0 (iPhone; CPU iPhone OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B179 Safari/7534.48.3' user_agent = parse(ua_string) - #print(user_agent) client = {} # Accessing user agent's browser attributes @@ -70,7 +69,7 @@ def getIP(self,request): ip = request.META.get('REMOTE_ADDR') if not ip_valid(ip): - print("GeoIP: IP address is NOT valid") + log.error("GeoIP: IP address is NOT valid") return None return ip @@ -99,5 +98,5 @@ def getGeo(self, ip): #self.reader.close() except Exception as e: - # print(e) + log.exception(e) return e diff --git a/src/_main_/utils/common.py b/src/_main_/utils/common.py index 1a20c11d3..54f231e16 100644 --- a/src/_main_/utils/common.py +++ b/src/_main_/utils/common.py @@ -273,11 +273,9 @@ def get_cookie(request, key): def set_cookie(response, key, value): # TODO - print(f"----- set_cookie: {response}") # set cookie on response before sending # cookie expiration set to 1yr MAX_AGE = 31536000 - response.set_cookie(key, value, MAX_AGE, samesite="Strict") diff --git a/src/_main_/utils/massenergize_logger/__init__.py b/src/_main_/utils/massenergize_logger/__init__.py index 494f920d3..f30ff1953 100644 --- a/src/_main_/utils/massenergize_logger/__init__.py +++ b/src/_main_/utils/massenergize_logger/__init__.py @@ -11,44 +11,38 @@ def __init__(self): self.logger = logging.getLogger(EnvConfig.get_logger_identifier()) - @run_in_background def _log(self, level, message, exception=None, extra={}): - if not EnvConfig.can_send_logs_to_cloudwatch(): - return - if exception: extra['exception'] = exception - # log through the python django logger which goes to Cloudwatch - if level >= logging.ERROR: - # send to cw - self.logger.log( - level, - message, - exc_info=True, - stack_info=True, - stacklevel=2, - extra=extra - ) - - # send to sentry + def _send_to_cw_and_log_files(): + # log through the python django logger which logs to Cloudwatch (for dev, canary, prod) + # and to log files (for other modes like test, local) + if level >= logging.ERROR: + self.logger.log( + level, + message, + exc_info=True, + stack_info=True, + extra=extra + ) + else: + self.logger.log(level, message, extra=extra) + + def _send_to_sentry(): if extra: with sentry_sdk.push_scope() as sentry_scope: for k,v in extra.items(): sentry_scope.set_extra(k, v) sentry_sdk.capture_message(message, level) - else: - # send to cw - self.logger.log(level, message, extra=extra) - # send log to sentry - if extra: - with sentry_sdk.push_scope() as sentry_scope: - for k,v in extra.items(): - sentry_scope.set_extra(k, v) - sentry_sdk.capture_message(message, level) - + # write to log files and send to cw + _send_to_cw_and_log_files() + + # don't send to sentry + if EnvConfig.can_send_logs_to_external_watchers(): + _send_to_sentry() def info(self, message, extra={}): self._log(logging.INFO, message, extra=extra) @@ -56,8 +50,8 @@ def info(self, message, extra={}): def error(self, message=None, exception=None, extra={}): self._log(logging.ERROR, message or str(exception), exception, extra) - def exception(self, exception): - self.error(str(exception), exception) + def exception(self, exception, message=None, extra={}): + self.error(message, exception, extra) return exception diff --git a/src/_main_/utils/metrics/__init__.py b/src/_main_/utils/metrics/__init__.py index 1aec426c1..c33d25411 100644 --- a/src/_main_/utils/metrics/__init__.py +++ b/src/_main_/utils/metrics/__init__.py @@ -57,7 +57,7 @@ def send_metric(func, execution_time, name_space=FUNCTION_LATENCY_NAMESPACE, ext def put_metric_data(name_space, metric_data): - if not EnvConfig.can_send_logs_to_cloudwatch(): + if not EnvConfig.can_send_logs_to_external_watchers(): return try: diff --git a/src/_main_/utils/stage/__init__.py b/src/_main_/utils/stage/__init__.py index fc03926e0..ce19b21bd 100644 --- a/src/_main_/utils/stage/__init__.py +++ b/src/_main_/utils/stage/__init__.py @@ -95,13 +95,15 @@ def is_local(self): def is_test(self): return self.name == "test" - def can_send_logs_to_cloudwatch(self): + def can_send_logs_to_external_watchers(self): return not (self.is_local() or self.is_test()) def get_logging_settings(self): - if self.is_local() or self.is_test(): + if self.is_local(): return get_local_logging_settings() + elif self.is_test(): + return get_testing_logging_settings() return get_default_logging_settings(self.name) @@ -151,13 +153,13 @@ def _set_api_run_info(self): is_docker_mode = False current_run_file_path = Path('.') / '.massenergize'/ 'current_run_info.json' - if current_run_file_path.exists(): + if not name and current_run_file_path.exists(): _current_run_info = load_json(current_run_file_path) name = _current_run_info.get('django_env', name) is_docker_mode = _current_run_info.get('is_docker_mode', is_docker_mode) else: load_dotenv() - name = os.getenv("DJANGO_ENV", "dev") + name = name or "dev" # use this value if name is None is_docker_mode = "DOCKER_CONTAINER" in os.environ name = name.lower() diff --git a/src/_main_/utils/stage/logging.py b/src/_main_/utils/stage/logging.py index 621f896bc..f66978053 100644 --- a/src/_main_/utils/stage/logging.py +++ b/src/_main_/utils/stage/logging.py @@ -7,6 +7,7 @@ BASE_DIR = os.getcwd() LOG_DIR = os.path.join(BASE_DIR, '.logs') LOG_FILE_NAME = "api.log" +TESTING_LOG_FILE_NAME = "api.testing.log" # Ensure the log directory exists if not os.path.exists(LOG_DIR): @@ -31,7 +32,7 @@ def get_default_logging_settings(stage): }, 'handlers': { 'external': { - 'level': 'DEBUG', + 'level': 'INFO', 'class': 'watchtower.CloudWatchLogHandler', 'log_group': f"/api/{stage}", 'stream_name': get_host_identifier(), @@ -42,7 +43,7 @@ def get_default_logging_settings(stage): 'filters': ['cid'], }, 'file': { - 'level': 'DEBUG', + 'level': 'INFO', 'class': 'logging.handlers.TimedRotatingFileHandler', 'filename': os.path.join(LOG_DIR, LOG_FILE_NAME), 'when': 'midnight', # Rotate logs at midnight @@ -107,18 +108,59 @@ def get_local_logging_settings(): }, "root": { "handlers": ["console", "file"], - "level": "WARNING", + "level": "INFO", }, 'loggers': { get_logger_name(): { 'handlers': ['console', 'file'], - 'level': 'DEBUG', + 'level': 'INFO', 'propagate': False, } } } +def get_testing_logging_settings(): + return { + "version": 1, + "disable_existing_loggers": False, + 'filters': { + 'cid': { + '()': 'cid.log.CidContextFilter' + }, + }, + 'formatters': { + 'detailed': { + 'format': '[request_id=%(cid)s] %(asctime)s %(levelname)s %(name)s %(message)s' + }, + }, + "handlers": { + "console": { + "class": "logging.StreamHandler", + }, + 'file': { + 'level': 'WARNING', + 'class': 'logging.handlers.TimedRotatingFileHandler', + 'filename': os.path.join(LOG_DIR, TESTING_LOG_FILE_NAME), + 'when': 'midnight', # Rotate logs at midnight + 'interval': 1, + 'backupCount': 7, # Keep last 7 days of logs + 'filters': ['cid'], + }, + }, + "root": { + "handlers": ["file"], + "level": "WARNING", + }, + 'loggers': { + get_logger_name(): { + 'handlers': ['file'], + 'level': 'DEBUG', + 'propagate': False, + } + } + } + def get_host_identifier(): try: ec2 = boto3.resource('ec2') diff --git a/src/api/handlers/misc.py b/src/api/handlers/misc.py index a818f78f1..78d788ea4 100644 --- a/src/api/handlers/misc.py +++ b/src/api/handlers/misc.py @@ -89,7 +89,6 @@ def backfill(self, request): def actions_report(self, request): context: Context = request.context args: dict = context.args - print("Got here") goal_info, err = self.service.actions_report(context, args) if err: return err diff --git a/src/api/services/auth.py b/src/api/services/auth.py index 4a5bff363..e1ec5851c 100644 --- a/src/api/services/auth.py +++ b/src/api/services/auth.py @@ -81,11 +81,10 @@ def login(self, context: Context): else: return None, None, CustomMassenergizeError("invalid_auth") except PermissionError: - log.error("not_an_admin", level="error") + log.error("not_an_admin", exception=PermissionError) return None, None, CustomMassenergizeError('not_an_admin') except Exception as e: - print(e) - log.error("Authentication Error", level="error") + log.exception(e) return None, None, CustomMassenergizeError(e) diff --git a/src/api/services/event.py b/src/api/services/event.py index c367f01b9..dddaf8ce3 100644 --- a/src/api/services/event.py +++ b/src/api/services/event.py @@ -134,7 +134,6 @@ def copy_event(self, context, args) -> Tuple[dict, MassEnergizeAPIError]: def list_recurring_event_exceptions(self, context, args) -> Tuple[list, MassEnergizeAPIError]: exceptions, err = self.store.list_recurring_event_exceptions(context, args) if err: - print(err) return None, err return exceptions, None diff --git a/src/api/services/userprofile.py b/src/api/services/userprofile.py index 04a3f15d2..b7f62a5da 100644 --- a/src/api/services/userprofile.py +++ b/src/api/services/userprofile.py @@ -332,7 +332,7 @@ def import_from_csv(self, context, args) -> Tuple[dict, MassEnergizeAPIError]: invalid_emails.append({"line":line, "email":email}) except Exception as e: - print("Error string: " + str(e)) + log.exception(e) return None, CustomMassenergizeError(e) if err: return None, err @@ -377,7 +377,6 @@ def import_from_list(self, context, args) -> Tuple[dict, MassEnergizeAPIError]: invalid_emails.append({"line":ix, "first_name":first_name, "last_name": last_name, "email":email}) except Exception as e: - print(str(e)) return None, CustomMassenergizeError(e) return {'invalidEmails': invalid_emails}, None except Exception as e: diff --git a/src/api/store/community.py b/src/api/store/community.py index 5743559b3..e1cbef68e 100644 --- a/src/api/store/community.py +++ b/src/api/store/community.py @@ -129,35 +129,35 @@ def _check_geography_unique(self, community, geography_type, loc): "Zipcode %s is already part of another geographic community %s." % (loc, check_community.name) ) - print(message) + log.info(message) raise Exception(message) elif geography_type == "CITY" and location.city == loc: message = ( "City %s is already part of another geographic community %s." % (loc, check_community.name) ) - print(message) + log.info(message) raise Exception(message) elif geography_type == "COUNTY" and location.county == loc: message = ( "County %s is already part of another geographic community %s." % (loc, check_community.name) ) - print(message) + log.info(message) raise Exception(message) elif geography_type == "STATE" and location.state == loc: message = ( "State (%s) is already part of another geographic community (%s)." % (loc, check_community.name) ) - print(message) + log.info(message) raise Exception(message) elif geography_type == "COUNTRY" and location.country == loc: message = ( "Country (%s) is already part of another geographic community (%s)." % (loc, check_community.name) ) - print(message) + log.info(message) raise Exception(message) def _are_locations_updated(self, geography_type, locations, community): @@ -202,7 +202,7 @@ def _update_locations(self, geography_type, locations, community): location_list = locations.replace(", ", ",").split( "," ) # passed as comma separated list - print("Community includes the following locations :" + str(location_list)) + log.info("Community includes the following locations :" + str(location_list)) for location in location_list: if geography_type == "ZIPCODE": if location[0].isdigit(): @@ -224,10 +224,10 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="ZIP_CODE_ONLY", zipcode=location, city=city ) - print("Zipcode " + location + " created for town " + city) + log.info("Zipcode " + location + " created for town " + city) else: loc = loc.first() - print("Zipcode " + location + " found for town " + city) + log.info("Zipcode " + location + " found for town " + city) self._check_geography_unique(community, geography_type, location) @@ -243,10 +243,10 @@ def _update_locations(self, geography_type, locations, community): zips = zipcodes.filter_by( city=town, state=state, zip_code_type="STANDARD" ) - print("Number of zipcodes = " + str(len(zips))) + log.info("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: for zip in zips: - print(zip) + log.info(zip) zipcode = zip["zip_code"] # get_or_create gives an error if multiple such locations exist (which can happen) @@ -262,17 +262,17 @@ def _update_locations(self, geography_type, locations, community): zipcode=location, city=town, ) - print("Zipcode " + zipcode + " created") + log.info("Zipcode " + zipcode + " created") else: loc = loc.first() - print("Zipcode " + zipcode + " found") + log.info("Zipcode " + zipcode + " found") self._check_geography_unique( community, geography_type, zipcode ) else: - print( + log.info( "No zipcodes found corresponding to town " + town + ", " @@ -296,7 +296,7 @@ def _update_locations(self, geography_type, locations, community): zips = zipcodes.filter_by( city=city, state=state, zip_code_type="STANDARD" ) - print("Number of zipcodes = " + str(len(zips))) + log.info("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: # get_or_create gives an error if multiple such locations exist (which can happen) # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) @@ -307,13 +307,13 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="CITY_ONLY", city=city, state=state ) - print("City " + city + " created") + log.info("City " + city + " created") else: loc = loc.first() - print("City " + city + " found") + log.info("City " + city + " found") else: - print( + log.info( "No zipcodes found corresponding to city " + city + ", " + state ) raise Exception( @@ -333,7 +333,7 @@ def _update_locations(self, geography_type, locations, community): zips = zipcodes.filter_by( county=county, state=state, zip_code_type="STANDARD" ) - print("Number of zipcodes = " + str(len(zips))) + log.info("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: # get_or_create gives an error if multiple such locations exist (which can happen) # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) @@ -344,13 +344,13 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="COUNTY_ONLY", county=county, state=state ) - print("County " + county + " created") + log.info("County " + county + " created") else: loc = loc.first() - print("County " + county + " found") + log.info("County " + county + " found") else: - print( + log.info( "No zipcodes found corresponding to county " + county + ", " @@ -369,7 +369,7 @@ def _update_locations(self, geography_type, locations, community): # check that this state is found in the zipcodes list state = location zips = zipcodes.filter_by(state=state, zip_code_type="STANDARD") - print("Number of zipcodes = " + str(len(zips))) + log.info("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: # get_or_create gives an error if multiple such locations exist (which can happen) # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) @@ -380,12 +380,12 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="STATE_ONLY", state=state ) - print("State " + state + " created") + log.info("State " + state + " created") else: loc = loc.first() - print("State " + state + " found") + log.info("State " + state + " found") else: - print("No zipcodes found corresponding to state " + location) + log.info("No zipcodes found corresponding to state " + location) raise Exception( "No zipcodes found corresponding to state " + location ) @@ -396,7 +396,7 @@ def _update_locations(self, geography_type, locations, community): # check that this state is found in the zipcodes list country = location zips = zipcodes.filter_by(country=country, zip_code_type="STANDARD") - print("Number of zipcodes = " + str(len(zips))) + log.info("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: # get_or_create gives an error if multiple such locations exist (which can happen) # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) @@ -407,12 +407,12 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="COUNTRY_ONLY", country=country ) - print("Country " + country + " created") + log.info("Country " + country + " created") else: loc = loc.first() - print("Country " + country + " found") + log.info("Country " + country + " found") else: - print("No zipcodes found corresponding to country " + location) + log.info("No zipcodes found corresponding to country " + location) raise Exception( "No zipcodes found corresponding to country " + location ) @@ -435,7 +435,7 @@ def _update_real_estate_units_with_community(self, community): is_deleted=False ) reus = RealEstateUnit.objects.all().select_related("address") - print("Updating " + str(reus.count()) + " RealEstateUnits") + log.info("Updating " + str(reus.count()) + " RealEstateUnits") # loop over profiles and realEstateUnits associated with them for userProfile in userProfiles: @@ -445,7 +445,7 @@ def _update_real_estate_units_with_community(self, community): zip = reu.address.zipcode if not isinstance(zip, str) or len(zip) != 5: address_string = str(reu.address) - print( + log.info( "REU invalid zipcode: address = " + address_string + " User " @@ -466,7 +466,7 @@ def _update_real_estate_units_with_community(self, community): reu.address.save() if is_reu_in_community(reu, community): - print( + log.info( "Adding the REU with zipcode " + zip + " to the community " @@ -477,7 +477,7 @@ def _update_real_estate_units_with_community(self, community): elif reu.community and reu.community.id == community.id: # this could be the case if the community was made smaller - print( + log.info( "REU not located in this community, but was labeled as belonging to the community" ) reu.community = None diff --git a/src/api/store/download.py b/src/api/store/download.py index 46729508b..7317d583f 100644 --- a/src/api/store/download.py +++ b/src/api/store/download.py @@ -1511,7 +1511,6 @@ def users_download( else: return EMPTY_DOWNLOAD, NotAuthorizedError() except Exception as e: - print(str(e)) log.exception(e) return EMPTY_DOWNLOAD, CustomMassenergizeError(e) diff --git a/src/api/store/event.py b/src/api/store/event.py index 799c4603a..34e0cbe2f 100644 --- a/src/api/store/event.py +++ b/src/api/store/event.py @@ -744,7 +744,7 @@ def update_recurring_event_date(self, context: Context, args) -> Tuple[dict, Mas exception.delete() except Exception as e: - print(str(e)) + log.exception(e) return CustomMassenergizeError(e) return None, None diff --git a/src/api/store/graph.py b/src/api/store/graph.py index 77cb673c4..929cf853d 100644 --- a/src/api/store/graph.py +++ b/src/api/store/graph.py @@ -443,7 +443,7 @@ def debug_data_fix(self) -> None: d.value = val d.save() if RUN_SERVER_LOCALLY: - print("WARNING - data_fix: Community: " + community.name + log.info("WARNING - data_fix: Community: " + community.name + ", Category: " + tag.name + ", Old: " + str(oldval) + ", New: " + str(val)) diff --git a/src/api/store/media_library.py b/src/api/store/media_library.py index dca79027f..97549fb28 100644 --- a/src/api/store/media_library.py +++ b/src/api/store/media_library.py @@ -110,7 +110,6 @@ def generate_hashes(self, args, context: Context): end = time.time() msg = f"Generated hashes for {generated} items in {end - start} seconds" - print(msg) return msg, None diff --git a/src/api/store/misc.py b/src/api/store/misc.py index d087d7e65..934cc4550 100644 --- a/src/api/store/misc.py +++ b/src/api/store/misc.py @@ -65,19 +65,16 @@ def navigation_menu_list( return None, CustomMassenergizeError(e) def actions_report(self, context: Context, args) -> Tuple[list, MassEnergizeAPIError]: - print("Actions report!") total = 0 total_wo_ccaction = 0 for c in Community.objects.filter(is_published=True): - print(c) actions = Action.objects.filter(community__id=c.id, is_published=True, is_deleted=False) total += actions.count() for action in actions: if action.calculator_action == None: total_wo_ccaction += 1 - print(action.title + " has no corresponding CCAction") - print("Total actions = "+str(total) + ", no CCAction ="+str(total_wo_ccaction)) - return None, None + log.info(f"Total actions = {str(total)}, no CCAction ={str(total_wo_ccaction)}") + return {"total": total, "total_wo_ccaction": total_wo_ccaction}, None def backfill(self, context: Context, args) -> Tuple[list, MassEnergizeAPIError]: return self.backfill_graph_default_data(context, args), None @@ -87,7 +84,7 @@ def backfill_subdomans(self): try: Subdomain(name=c.subdomain, in_use=True, community=c).save() except Exception as e: - print(e) + log.exception(e, f"could not backfill subdomain: {c.subdomain}") def backfill_teams( self, context: Context, args @@ -193,7 +190,7 @@ def backfill_graph_default_data(self, context: Context, args): d.value = val d.save() - print( + log.info( "Backfill: Community: " + community.name + ", Category: " @@ -217,12 +214,12 @@ def backfill_real_estate_units(self, context: Context, args): # This defines what geographic community, if any gets credit # For now, check for zip code reu_all = RealEstateUnit.objects.all() - print("Number of real estate units:" + str(reu_all.count())) + log.info("Number of real estate units:" + str(reu_all.count())) userProfiles = UserProfile.objects.prefetch_related( "real_estate_units" ).filter(is_deleted=False) - print("number of user profiles:" + str(userProfiles.count())) + log.info("number of user profiles:" + str(userProfiles.count())) # loop over profiles and realEstateUnits associated with them for userProfile in userProfiles: @@ -234,8 +231,6 @@ def backfill_real_estate_units(self, context: Context, args): reus.count(), userProfile.created_at.strftime("%Y-%m-%d"), ) - print(msg) - for reu in reus: street = unit_number = city = county = state = zip = "" loc = reu.location # a JSON field @@ -244,7 +239,7 @@ def backfill_real_estate_units(self, context: Context, args): if loc: # if not isinstance(loc,str): # # one odd case in dev DB, looked like a Dict - # print("REU location not a string: "+str(loc)+" Type="+str(type(loc))) + # log.info("REU location not a string: "+str(loc)+" Type="+str(type(loc))) # loc = loc["street"] loc_parts = split_location_string(loc) @@ -259,9 +254,6 @@ def backfill_real_estate_units(self, context: Context, args): state = loc_parts[2].upper() zip = loc_parts[3].strip() if not zip or (len(zip) != 5 and len(zip) != 10): - print( - "Invalid zipcode: " + zip + ", setting to 00000" - ) zip = "00000" elif len(zip) == 10: zip = zip[0:5] @@ -275,14 +267,13 @@ def backfill_real_estate_units(self, context: Context, args): city = ZIPCODE_FIXES[entry]["city"] state = ZIPCODE_FIXES[entry].get("state", "MA") - print("Zipcode assigned " + zip) # create the Location for the RealEstateUnit location_type, valid = check_location( street, unit_number, city, state, zip ) if not valid: - print("check_location returns: " + location_type) + log.info("check_location returns: " + location_type) continue # newloc, created = Location.objects.get_or_create( @@ -305,10 +296,10 @@ def backfill_real_estate_units(self, context: Context, args): county=county, state=state, ) - print("Zipcode " + zip + " created for town " + city) + log.info("Zipcode " + zip + " created for town " + city) else: newloc = newloc.first() - print("Zipcode " + zip + " found for town " + city) + log.info("Zipcode " + zip + " found for town " + city) reu.address = newloc reu.save() @@ -332,7 +323,7 @@ def backfill_real_estate_units(self, context: Context, args): # no location was stored? if zip == "00000": - print("No location found for RealEstateUnit " + str(reu)) + log.info("No location found for RealEstateUnit " + str(reu)) location_type = "ZIP_CODE_ONLY" newloc, created = Location.objects.get_or_create( @@ -345,16 +336,16 @@ def backfill_real_estate_units(self, context: Context, args): state=state, ) if created: - print("Location with zipcode " + zip + " created") + log.info("Location with zipcode " + zip + " created") else: - print("Location with zipcode " + zip + " found") + log.info("Location with zipcode " + zip + " found") reu.address = newloc reu.save() # determine which, if any, community this household is actually in community = find_reu_community(reu) if community: - print( + log.info( "Adding the REU with zipcode " + zip + " to the community " @@ -363,7 +354,7 @@ def backfill_real_estate_units(self, context: Context, args): reu.community = community elif reu.community: - print( + log.info( "REU not located in any community, but was labeled as belonging to the community " + reu.community.name ) @@ -477,7 +468,7 @@ def list_commonly_used_icons(self): sorted_keys = sorted(common_icons, key=common_icons.get, reverse=True) for key in sorted_keys: - print(str(key) + ": " + str(common_icons[key])) + log.info(str(key) + ": " + str(common_icons[key])) def load_menu_items(self, context, args): try: diff --git a/src/api/store/team.py b/src/api/store/team.py index 46fba3c45..ed2ea86ad 100644 --- a/src/api/store/team.py +++ b/src/api/store/team.py @@ -381,7 +381,6 @@ def update_team(self, context, args) -> Tuple[dict, MassEnergizeAPIError]: # ---------------------------------------------------------------- return team, None except Exception as e: - print(str(e)) log.exception(e) return None, CustomMassenergizeError(e) diff --git a/src/api/store/userprofile.py b/src/api/store/userprofile.py index aa6b5c700..ccd8859dd 100644 --- a/src/api/store/userprofile.py +++ b/src/api/store/userprofile.py @@ -78,9 +78,9 @@ def _get_or_create_reu_location(args, user=None): ) if created: - print("Location with zipcode " , zipcode , " created for user " , user.preferred_name) + log.info(f"Location with zipcode, {zipcode} , created for user, {user.preferred_name}") else: - print("Location with zipcode " , zipcode , " found for user " , user.preferred_name) + log.info(f"Location with zipcode, {zipcode} , found for user, {user.preferred_name}") return reuloc def _update_action_data_totals(action, household, delta): @@ -443,7 +443,6 @@ def add_household(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIEr return reu, None except Exception as e: - print("=== error from add_household ===", e) log.exception(e) return None, CustomMassenergizeError(e) @@ -468,7 +467,8 @@ def edit_household(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIE verbose = DEBUG community = find_reu_community(reu, verbose) if community: - if verbose: print("Updating the REU with zipcode " + reu.address.zipcode + " to the community " + community.name) + if verbose: + log.info("Updating the REU with zipcode " + reu.address.zipcode + " to the community " + community.name) reu.community = community reu.save() @@ -490,7 +490,6 @@ def list_users(self, context,community_id) -> Tuple[list, MassEnergizeAPIError]: community, err = get_community(community_id) if not community: - print(err) return [], None if not is_admin_of_community(context, community_id): @@ -501,7 +500,6 @@ def list_publicview(self, context, args) -> Tuple[list, MassEnergizeAPIError]: community_id = args.pop('community_id', None) community, err = get_community(community_id) if not community: - print(err) return [], None LEADERBOARD_MINIMUM = 100 @@ -849,7 +847,6 @@ def list_users_for_community_admin(self, context: Context, args) -> Tuple[list, return users.distinct(), None elif not community: - print(err) return [], None if not is_admin_of_community(context, community_id): diff --git a/src/api/utils/api_utils.py b/src/api/utils/api_utils.py index 3a8e47466..0b726e8ef 100644 --- a/src/api/utils/api_utils.py +++ b/src/api/utils/api_utils.py @@ -7,10 +7,8 @@ ContactUsPageSettings, DonatePageSettings, EventsPageSettings, ImpactPageSettings, Media, Menu, \ TeamsPageSettings, TestimonialsPageSettings, UserProfile, \ VendorsPageSettings -import pyshorteners - from _main_.utils.constants import COMMUNITY_URL_ROOT - +from _main_.utils.massenergize_logger import log def is_admin_of_community(context, community_id): # super admins are admins of any community @@ -312,5 +310,5 @@ def get_list_of_internal_links(is_footer=False): return internal_links, None except Exception as e: - print(f"Error: {str(e)}") + log.exception(e) return None, str(e) diff --git a/src/apps__campaigns/create_campaign_template.py b/src/apps__campaigns/create_campaign_template.py index 99c6e9951..1114e52c4 100644 --- a/src/apps__campaigns/create_campaign_template.py +++ b/src/apps__campaigns/create_campaign_template.py @@ -25,7 +25,7 @@ TechnologyVendor, ) from database.models import Community, Event, Media, Testimonial, UserProfile, Vendor - +from _main_.utils.massenergize_logger import log TEMPLATE_TITLE = "Template Campaign" @@ -38,7 +38,7 @@ def create_media(image_path): media.name = f"Test media for {image_path}" return media except Exception as e: - print(f"Error creating media: {str(e)}") + log.exception(e, message="Error creating media") return None @@ -85,7 +85,7 @@ def get_3_communities(): comm.append(community) return comm except Exception as e: - print(f"Error creating communities: {str(e)}") + log.exception(e, "Error creating communities") return [] @@ -118,7 +118,7 @@ def create_campaign_technology_overview(technology_id): campaign_technology_overview.image = item["image"] campaign_technology_overview.save() except Exception as e: - print(f"Error creating campaign technology overview: {str(e)}") + log.exception(e, message=f"Error creating campaign technology overview") def create_technology_coaches(technology_id): @@ -154,7 +154,7 @@ def create_technology_coaches(technology_id): coach.image = item["image"] coach.save() except Exception as e: - print(f"Error creating technology coaches: {str(e)}") + log.exception(e, message="Error creating technology coaches: {str(e)}") def create_technology_vendors(technology_id): @@ -168,7 +168,7 @@ def create_technology_vendors(technology_id): tech_vendor.vendor = vendor tech_vendor.save() except Exception as e: - print(f"Error creating technology vendors: {str(e)}") + log.exception(e, message="Error creating technology vendors") def create_technology_events(campaign): @@ -222,7 +222,7 @@ def create_technology_events(campaign): campaign_event.campaign_technology = tech campaign_event.save() except Exception as e: - print(f"Error creating Campaign events: {str(e)}") + log.exception(e, message=f"Error creating Campaign events: {str(e)}") @@ -245,7 +245,7 @@ def create_technology(name, image=None, description=None): return technology except Exception as e: - print(f"Error creating technology: {str(e)}") + log.exception(e, message="Error creating technology") def create_campaign_technology_testimonial(campaign_technology_id): @@ -309,7 +309,7 @@ def create_campaign_technology_testimonial(campaign_technology_id): campaign_technology_view.save() except Exception as e: - print(f"An error occurred while creating testimonials: {str(e)}") + log.exception(e, message="An error occurred while creating testimonials") def create_campaign_partners(campaign): @@ -382,7 +382,7 @@ def create_campaign_Managers(campaign): campaign_manager.contact = user["contact"] campaign_manager.save() except Exception as e: - print(f"An error occurred while creating campaign managers: {str(e)}") + log.exception(e, message="An error occurred while creating campaign managers") def create_campaign_configuration(campaign): @@ -429,7 +429,7 @@ def create_campaign_events(campaign): campaign_event.campaign_technology = tech campaign_event.save() except Exception as e: - print(f"An error occurred while creating events: {str(e)}") + log.exception(e, message="An error occurred while creating events") @@ -470,7 +470,7 @@ def create_template_campaign(): return campaign except Exception as e: - print(f"An error occurred while creating template campaign: {str(e)}") + log.exception(e, message="An error occurred while creating template campaign") return None @@ -505,7 +505,7 @@ def create_template_campaign_technology(campaign_id): create_campaign_technology_testimonial(campaign_technology.id) except Exception as e: - print(f"An error occurred while creating template campaign technologies: {str(e)}") + log.exception(e, message="An error occurred while creating template campaign technologies") @@ -524,4 +524,4 @@ def run(): print("Template Campaign Created Successfully !!!") return except Exception as e: - print(f"An error occurred while creating the template campaign: {str(e)}") + log.exception(e, message="An error occurred while creating the template campaign") diff --git a/src/carbon_calculator/CCDefaults.py b/src/carbon_calculator/CCDefaults.py index ee57f92c4..d24d3df70 100644 --- a/src/carbon_calculator/CCDefaults.py +++ b/src/carbon_calculator/CCDefaults.py @@ -3,6 +3,7 @@ from datetime import datetime from django.utils import timezone import csv +from _main_.utils.massenergize_logger import log current_tz = timezone.get_current_timezone() @@ -87,8 +88,7 @@ def loadDefaults(self): self.DefaultsByLocality[c.locality][c.variable]["values"].append(c.value) except Exception as e: - print(str(e)) - print("CalcDefault initialization skipped") + log.exception(e, "CalcDefault initialization skipped") def getDefault(self, locality, variable, date, default=None): # load default values if they haven't yet been loaded diff --git a/src/carbon_calculator/carbonCalculator.py b/src/carbon_calculator/carbonCalculator.py index 9d5d270fc..20e53b71b 100644 --- a/src/carbon_calculator/carbonCalculator.py +++ b/src/carbon_calculator/carbonCalculator.py @@ -5,6 +5,7 @@ import os import pytz from _main_.settings import BASE_DIR, RUN_SERVER_LOCALLY +from _main_.utils.massenergize_logger import log from datetime import datetime, date import time from .models import Action, Question, CarbonCalculatorMedia, Version, Category, Subcategory @@ -196,7 +197,7 @@ def __init__(self, reset=False) : print("Carbon Calculator initialization time: "+str(end - start)+" seconds") except Exception as e: - print(str(e)) + log.exception(e) print("Calculator initialization skipped") # query actions @@ -405,7 +406,7 @@ def import_helper(self, inputs, string, status, method): csvfile.close() return True except Exception as e: - print(str(e)) + log.exception(e) return False @@ -491,8 +492,7 @@ def ImportQuestions(self, questionsFile): csvfile.close() return True except Exception as e: - print(str(e)) - print('Error importing Carbon Calculator questions') + log.exception(e, message='Error importing Carbon Calculator questions') return False def ImportActions(self, actionsFile): @@ -543,8 +543,7 @@ def ImportActions(self, actionsFile): csvfile.close() return True except Exception as e: - print(str(e)) - print('Error importing Carbon Calculator actions') + log.exception(e, 'Error importing Carbon Calculator actions') return False diff --git a/src/database/models.py b/src/database/models.py index b0a1369f1..8bc59344e 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -30,6 +30,7 @@ from django.forms.models import model_to_dict from carbon_calculator.models import Action as CCAction from carbon_calculator.carbonCalculator import AverageImpact +from _main_.utils.massenergize_logger import log CHOICES = json_loader("./database/raw_data/other/databaseFieldChoices.json") ZIP_CODE_AND_STATES = json_loader("./database/raw_data/other/states.json") @@ -1108,7 +1109,7 @@ def update_visit_log(self, date_time): else: self.visit_log.append(date) except Exception as e: - print(e) + log.exception(e) return None, e def full_json(self): @@ -1372,7 +1373,7 @@ def update_visit_log(self, date_time): self.visit_log.append(date) except Exception as e: - print(e) + log.exception(e) return None, e def simple_json(self): diff --git a/src/task_queue/database_tasks/contents_spacing_correction.py b/src/task_queue/database_tasks/contents_spacing_correction.py index 50de590fa..6ebf4fa71 100644 --- a/src/task_queue/database_tasks/contents_spacing_correction.py +++ b/src/task_queue/database_tasks/contents_spacing_correction.py @@ -106,7 +106,6 @@ def process_spacing_data(task=None): return True except Exception as e: - print(str(e)) log.exception(e) return False diff --git a/src/task_queue/database_tasks/update_actions_content.py b/src/task_queue/database_tasks/update_actions_content.py index ed5588282..140c09b4f 100644 --- a/src/task_queue/database_tasks/update_actions_content.py +++ b/src/task_queue/database_tasks/update_actions_content.py @@ -108,7 +108,7 @@ def update_actions_content(task=None): # set calculator_action, ccAction = ccActions.filter(name=ccActionName) if not ccAction: - print("Carbon calculator action '"+ccActionName+"' does not exist") + log.info(f"Carbon calculator action {ccActionName} does not exist") continue action.calculator_action = ccAction.first() action.save() @@ -122,7 +122,6 @@ def update_actions_content(task=None): return True except Exception as e: - print(str(e)) log.exception(e) return False diff --git a/src/task_queue/nudges/cadmin_events_nudge.py b/src/task_queue/nudges/cadmin_events_nudge.py index 811f00bcb..e5ba4d4c9 100644 --- a/src/task_queue/nudges/cadmin_events_nudge.py +++ b/src/task_queue/nudges/cadmin_events_nudge.py @@ -13,7 +13,7 @@ import pytz from django.db.models import Q from dateutil.relativedelta import relativedelta - +from _main_.utils.massenergize_logger import log from task_queue.helpers import get_event_location WEEKLY = "weekly" @@ -117,7 +117,7 @@ def get_email_list(admins): name = admin.get("name") email = admin.get("email") if not name or not email: - print("Missing name or email for admin: " + str(admin)) + log.info("Missing name or email for admin: " + str(admin)) continue # BHN - fixed crash on this next line if communication_prefs didn't exist @@ -198,13 +198,13 @@ def send_events_nudge(task=None): for name, email in email_list.items(): stat = send_events_report(name, email, event_list) if not stat: - print("send_events_report error return") + log.error("send_events_report error return") return False admins_emailed.append(email) update_last_notification_dates(admins_emailed) return True except Exception as e: - print("Community admin nudge exception: " + str(e)) + log.exception(e) return False @@ -224,5 +224,5 @@ def send_events_report(name, email, event_list): send_massenergize_email_with_attachments(WEEKLY_EVENTS_NUDGE_TEMPLATE, data, [email], None, None, None) return True except Exception as e: - print("send_events_report exception: " + str(e)) + log.exception(e) return False diff --git a/src/task_queue/nudges/postmark_sender_signature.py b/src/task_queue/nudges/postmark_sender_signature.py index 1388407bf..aa44ab8c8 100644 --- a/src/task_queue/nudges/postmark_sender_signature.py +++ b/src/task_queue/nudges/postmark_sender_signature.py @@ -4,6 +4,7 @@ get_sender_signature_info, resend_signature_confirmation, ) +from _main_.utils.massenergize_logger import log #from _main_.utils.feature_flag_keys import POSTMARK_COMMUNITY_EMAIL_SENDER_SIGNATURE_FF #from database.models import Community, FeatureFlag from database.models import Community @@ -44,9 +45,9 @@ def collect_and_create_signatures(task=None): "nudge_count": postmark_info.get("nudge_count", 0) + 1, } else: - print(f"ERROR Resending Confirmation to {community.name}: ", res.json()) + log.error(f"ERROR Resending Confirmation to {community.name}: ", extra={'data': str(response.json())}) else: - print(f"ERROR getting Contact Info of {community.name}: ", response.json()) + log.error(f"ERROR getting Contact Info of {community.name}: ", extra={'data': str(response.json())}) else: response = add_sender_signature(email, alias, community.owner_name, community.name) if response.status_code == 200: @@ -62,5 +63,5 @@ def collect_and_create_signatures(task=None): community.save() else: - print(f"ERROR adding signature for {community.name}: ", response.json()) + log.error(f"ERROR adding signature for {community.name}: ", extra={'data': str(response.json())}) return True diff --git a/src/task_queue/nudges/user_event_nudge.py b/src/task_queue/nudges/user_event_nudge.py index 34cca8219..60775a188 100644 --- a/src/task_queue/nudges/user_event_nudge.py +++ b/src/task_queue/nudges/user_event_nudge.py @@ -18,6 +18,7 @@ from django.utils import timezone from task_queue.helpers import get_event_location +from _main_.utils.massenergize_logger import log WEEKLY = "per_week" BI_WEEKLY = "biweekly" @@ -147,7 +148,7 @@ def is_event_eligible(event, community_id, task=None): return False except Exception as e: - print(f"is_event_eligible exception - (event:{event.name}|| community:{community_id}): " + str(e)) + log.exception(e, exta={"event_name": event.name, "community": community_id}) return False @@ -218,7 +219,7 @@ def prepare_events_email_data(events): return data -def community_has_altered_flow(community, feature_flag_key) -> bool: +def community_has_altered_flow(community: Community, feature_flag_key) -> bool: try: today = timezone.now().today().date() community_nudge_settings = CommunityNotificationSetting.objects.filter(community=community, @@ -244,7 +245,7 @@ def community_has_altered_flow(community, feature_flag_key) -> bool: return False return True except Exception as e: - print(f"community_has_altered_flow exception - ({community.name}): " + str(e)) + log.exception(e, message="community_has_altered_flow", extra={"community": community.subdomain}) return False @@ -266,10 +267,10 @@ def send_events_report_email(name, email, event_list, comm, login_method=""): data["community"] = comm.name from_email = get_sender_email(comm.id) send_massenergize_email_with_attachments(USER_EVENTS_NUDGE_TEMPLATE, data, [email], None, None, from_email) - print("Email sent to " + email) + log.info("Email sent to " + email) return True except Exception as e: - print("send_events_report exception: " + str(e)) + log.exception(e, extra={"name": name, "email": email, "evenet_list": event_list}) return False @@ -279,15 +280,15 @@ def send_automated_nudge(events, user, community): email = user.email login_method = (user.user_info or {}).get("login_method") or "" if not name or not email: - print("Missing name or email for user: " + str(user)) + log.info("Missing name or email for user: " + str(user)) return False user_is_ready_for_nudge = should_user_get_nudged(user) if user_is_ready_for_nudge: - print("sending nudge to " + email) + log.info("sending nudge to " + email) is_sent = send_events_report_email(name, email, events, community, login_method) if not is_sent: - print(f"**** Failed to send email to {name} for community {community.name} ****") + log.error(f"**** Failed to send email to {name} for community {community.name} ****") return False update_last_notification_dates(email) return True @@ -300,7 +301,7 @@ def send_user_requested_nudge(events, user, community): login_method = (user.user_info or {}).get("login_method") or "" is_sent = send_events_report_email(name, email, events, community, login_method) if not is_sent: - print(f"**** Failed to send email to {name} for community {community.name} ****") + log.error(f"Failed to send email to {name} for community {community.name}") return False return True @@ -362,5 +363,5 @@ def prepare_user_events_nudge(task=None, email=None, community_id=None): return True except Exception as e: - print("Community member nudge exception: " + str(e)) + log.exception(e,message="Community member nudge exception") return False diff --git a/src/task_queue/views.py b/src/task_queue/views.py index 199b79f5f..41d7fa61f 100644 --- a/src/task_queue/views.py +++ b/src/task_queue/views.py @@ -17,6 +17,7 @@ from django.db.models import Q from django.core.exceptions import ObjectDoesNotExist from carbon_calculator.carbonCalculator import AverageImpact +from _main_.utils.massenergize_logger import log today = datetime.datetime.utcnow().replace(tzinfo=utc) one_week_ago = today - timezone.timedelta(days=7) @@ -379,7 +380,7 @@ def create_snapshots(task=None): return "Success" except Exception as e: - print("Community snapshot exception: " + str(e)) + log.exception(e) return "Failure" def send_mou_email(email, name): @@ -439,7 +440,7 @@ def send_admin_mou_notification(task=None): needs_to_sign_mou = last_record.signed_at <= a_year_ago else: if not IS_PROD: - print(admin_name + " has no MOU acceptance recorded") + log.info(admin_name + " has no MOU acceptance recorded") needs_to_sign_mou = True # If it's time to notify the admin again, then add a new notification timestamp to their policy record @@ -456,7 +457,7 @@ def send_admin_mou_notification(task=None): not last_date_of_notification ): # If for some reason notification date has never been recorded if not IS_PROD: - print("Sending first MOU notification") + log.info("Sending first MOU notification") send_mou_email(admin.email, admin_name) update_records(last=last_record, notices=notices) @@ -467,16 +468,17 @@ def send_admin_mou_notification(task=None): if not_notified_recently: #only notify if its been more than a month of notifying if not IS_PROD: - print("Overdue: Sending another notification") + log.info("Overdue: Sending another notification") send_mou_email(admin.email, admin_name) update_records(last=last_record, notices=notices) - except ObjectDoesNotExist: + except ObjectDoesNotExist as e: + log.exception(e) # If no MOU record exists for the admin, this means the first time they need to sign the MOU if not IS_PROD: - print("Except: Sending first admin MOU notificaiton to " + admin_name) + log.info("Except: Sending first admin MOU notificaiton to " + admin_name) send_mou_email(admin.email, admin_name) # Record the current notification timestamp diff --git a/src/website/views.py b/src/website/views.py index c38e6817c..3b337583d 100644 --- a/src/website/views.py +++ b/src/website/views.py @@ -264,7 +264,7 @@ def _get_is_sandbox(request): ps = par.split("=") if ps[0].lower() == "sandbox": if len(ps) < 2 or ps[1].lower() == "true": - print("it is the sandbox") + log.info("it is the sandbox") return True return False # sandbox=false return False # sandbox not specified