diff --git a/src/Makefile b/src/Makefile index 214c892c4..d979ca043 100644 --- a/src/Makefile +++ b/src/Makefile @@ -118,7 +118,7 @@ celery-remote: celery -A _main_.celery.app worker -l info full-celery-local: - make -j 2 start celery + DJANGO_ENV="local" make -j 2 start celery .PHONY: full-celery-local beat-remote: diff --git a/src/_main_/settings.py b/src/_main_/settings.py index ea357b045..9d7076d6e 100644 --- a/src/_main_/settings.py +++ b/src/_main_/settings.py @@ -13,7 +13,6 @@ import os import firebase_admin from firebase_admin import credentials -from dotenv import load_dotenv import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration from sentry_sdk.integrations.celery import CeleryIntegration diff --git a/src/api/handlers/deviceprofile.py b/src/api/handlers/deviceprofile.py index 5137b0d8d..f6f4f9e53 100644 --- a/src/api/handlers/deviceprofile.py +++ b/src/api/handlers/deviceprofile.py @@ -110,7 +110,7 @@ def platform_metrics(self, request): if metric: if metric == "anonymous_users": - metric, err = self.service.metric_anonymous_users(context, args) + metric, err = self.service.metric_anonymous_users() elif community_id and metric == "anonymous_community_users": metric, err = self.service.metric_anonymous_community_users(community_id) diff --git a/src/api/handlers/misc.py b/src/api/handlers/misc.py index f3f3814f7..c410a2e6a 100644 --- a/src/api/handlers/misc.py +++ b/src/api/handlers/misc.py @@ -23,7 +23,6 @@ def registerRoutes(self) -> None: self.add("/menus.remake", self.remake_navigation_menu) self.add("/menus.list", self.navigation_menu_list) self.add("/user.portal.menu.list", self.load_menu_items) - self.add("/data.backfill", self.backfill) self.add("/data.carbonEquivalency.create", self.create_carbon_equivalency) self.add("/data.carbonEquivalency.update", self.update_carbon_equivalency) self.add("/data.carbonEquivalency.get", self.get_carbon_equivalencies) @@ -81,14 +80,6 @@ def navigation_menu_list(self, request): return err return MassenergizeResponse(data=goal_info) - def backfill(self, request): - context: Context = request.context - args: dict = context.args - goal_info, err = self.service.backfill(context, args) - if err: - return err - return MassenergizeResponse(data=goal_info) - def actions_report(self, request): context: Context = request.context args: dict = context.args diff --git a/src/api/handlers/userprofile.py b/src/api/handlers/userprofile.py index 13fc05ab3..df02ef061 100644 --- a/src/api/handlers/userprofile.py +++ b/src/api/handlers/userprofile.py @@ -1,14 +1,10 @@ """Handler file for all routes pertaining to users""" -from functools import wraps -from _main_.utils import context from _main_.utils.route_handler import RouteHandler from api.services.userprofile import UserService from _main_.utils.massenergize_response import MassenergizeResponse from _main_.utils.massenergize_errors import NotAuthorizedError from _main_.utils.context import Context from api.decorators import admins_only, super_admins_only, login_required -from api.store.common import create_pdf_from_rich_text -from database.models import Policy class UserHandler(RouteHandler): diff --git a/src/api/services/deviceprofile.py b/src/api/services/deviceprofile.py index 56886f501..6e1646866 100644 --- a/src/api/services/deviceprofile.py +++ b/src/api/services/deviceprofile.py @@ -1,8 +1,6 @@ -from _main_.utils.massenergize_errors import CustomMassenergizeError, MassEnergizeAPIError -from _main_.utils.common import serialize, serialize_all +from _main_.utils.massenergize_errors import MassEnergizeAPIError +from _main_.utils.common import serialize from api.store.deviceprofile import DeviceStore -from _main_.utils.context import Context -from _main_.utils.massenergize_logger import log from typing import Tuple class DeviceService: diff --git a/src/api/services/misc.py b/src/api/services/misc.py index 8c8d0f3fd..60fe85459 100644 --- a/src/api/services/misc.py +++ b/src/api/services/misc.py @@ -54,11 +54,6 @@ def navigation_menu_list( return None, err return serialize_all(main_menu_items), None - def backfill(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIError]: - result, err = self.store.backfill(context, args) - if err: - return None, err - return result, None def actions_report(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIError]: result, err = self.store.actions_report(context, args) if err: diff --git a/src/api/services/team.py b/src/api/services/team.py index 2558767d4..880325a3a 100644 --- a/src/api/services/team.py +++ b/src/api/services/team.py @@ -3,7 +3,6 @@ from _main_.utils.pagination import paginate from api.store.team import TeamStore from api.store.message import MessageStore -from api.utils.api_utils import get_sender_email from api.utils.filter_functions import sort_items from database.models import TeamMember from _main_.utils.context import Context diff --git a/src/api/services/technology.py b/src/api/services/technology.py index 496e33160..942bb1413 100644 --- a/src/api/services/technology.py +++ b/src/api/services/technology.py @@ -121,13 +121,6 @@ def list_technology_vendors(self, context: Context, args) -> Tuple[list, MassEn return serialize_all(res), None - def list_technologies_for_admin(self, context: Context, args) -> Tuple[list, MassEnergizeAPIError]: - technologies, err = self.store.list_technologies_for_admin(context, args) - if err: - return None, err - return serialize_all(technologies), None - - def create_technology_deal(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIError]: res, err = self.store.create_technology_deal(context, args) if err: diff --git a/src/api/services/userprofile.py b/src/api/services/userprofile.py index 04a3f15d2..b8ccb3a1e 100644 --- a/src/api/services/userprofile.py +++ b/src/api/services/userprofile.py @@ -1,7 +1,6 @@ from _main_.utils.massenergize_errors import CustomMassenergizeError, MassEnergizeAPIError from _main_.utils.common import serialize, serialize_all from _main_.utils.pagination import paginate -from api.decorators import login_required from api.store.userprofile import UserStore from _main_.utils.context import Context from _main_.utils.emailer.send_email import send_massenergize_rich_email diff --git a/src/api/store/admin.py b/src/api/store/admin.py index f6190a163..4f0d7083d 100644 --- a/src/api/store/admin.py +++ b/src/api/store/admin.py @@ -1,7 +1,7 @@ from _main_.utils.utils import is_not_null from api.utils.api_utils import is_admin_of_community from api.utils.filter_functions import get_super_admins_filter_params -from database.models import UserProfile, CommunityAdminGroup, Media, UserProfile, Message +from database.models import UserProfile, CommunityAdminGroup, Media, Message from _main_.utils.massenergize_errors import MassEnergizeAPIError, CustomMassenergizeError, NotAuthorizedError from _main_.utils.context import Context from .utils import get_community, get_user, get_community_or_die, unique_media_filename diff --git a/src/api/store/common.py b/src/api/store/common.py index f9d277732..64b427cb9 100644 --- a/src/api/store/common.py +++ b/src/api/store/common.py @@ -12,10 +12,10 @@ from _main_.settings import AWS_S3_REGION_NAME from _main_.utils.common import custom_timezone_info, serialize, serialize_all from api.constants import CSV_FIELD_NAMES -from api.store.utils import getCarbonScoreFromActionRel from carbon_calculator.models import Action +from database.utils.common import calculate_hash_for_bucket_item from database.models import Community, CommunityAdminGroup, Event, Media, Team, UserActionRel -from database.utils.common import calculate_hash_for_bucket_item, get_image_size_from_bucket +from carbon_calculator.carbonCalculator import getCarbonImpact s3 = boto3.client("s3", region_name=AWS_S3_REGION_NAME) @@ -104,7 +104,7 @@ def count_action_completed_and_todos(**kwargs): for completed_action in completed_actions: action_id = completed_action.action.id action_name = completed_action.action.title - action_carbon = getCarbonScoreFromActionRel(completed_action) + action_carbon = getCarbonImpact(completed_action) done = 1 if completed_action.status == "DONE" else 0 todo = 1 if completed_action.status == "TODO" else 0 diff --git a/src/api/store/community.py b/src/api/store/community.py index 503704098..245dd3390 100644 --- a/src/api/store/community.py +++ b/src/api/store/community.py @@ -229,23 +229,21 @@ def _update_locations(self, geography_type, locations, community): zipcode = zipcodes.matching(location) if len(zipcode) > 0: city = zipcode[0].get("city", None) + county = zipcode[0].get("county", None) + state = zipcode[0].get("state", None) else: raise Exception("No zip code entry found for zip=" + location) # get_or_create gives an error if multiple such locations exist (which can happen) - # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) loc = Location.objects.filter( - location_type="ZIP_CODE_ONLY", zipcode=location, city=city + location_type="FULL_ADDRESS", zipcode=location, city=city, county=county, state=state ) if not loc: loc = Location.objects.create( - location_type="ZIP_CODE_ONLY", zipcode=location, city=city + location_type="FULL_ADDRESS", zipcode=location, city=city, county=county, state=state ) - print("Zipcode " + location + " created for town " + city) else: loc = loc.first() - print("Zipcode " + location + " found for town " + city) - self._check_geography_unique(community, geography_type, location) else: @@ -260,47 +258,28 @@ def _update_locations(self, geography_type, locations, community): zips = zipcodes.filter_by( city=town, state=state, zip_code_type="STANDARD" ) - print("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: for zip in zips: - print(zip) - zipcode = zip["zip_code"] + zipcode = zip.get("zip_code") + county = zip.get("county") # get_or_create gives an error if multiple such locations exist (which can happen) # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) loc = Location.objects.filter( - location_type="ZIP_CODE_ONLY", - zipcode=location, - city=town, + location_type="FULL_ADDRESS", zipcode=zipcode, city=town, county=county, state=state ) if not loc: loc = Location.objects.create( - location_type="ZIP_CODE_ONLY", - zipcode=location, - city=town, + location_type="FULL_ADDRESS", zipcode=location, city=city, county=county, state=state ) - print("Zipcode " + zipcode + " created") else: loc = loc.first() - print("Zipcode " + zipcode + " found") - - self._check_geography_unique( - community, geography_type, zipcode - ) + self._check_geography_unique(community, geography_type, zipcode) else: - print( - "No zipcodes found corresponding to town " - + town - + ", " - + state - ) - raise Exception( - "No zipcodes found corresponding to city " - + town - + ", " - + state - ) + msg = "No zipcodes found corresponding to town " + town + ", " + state + raise Exception(msg) + elif geography_type == "CITY": # check that this city is found in the zipcodes list ss = location.split("-") @@ -313,29 +292,21 @@ def _update_locations(self, geography_type, locations, community): zips = zipcodes.filter_by( city=city, state=state, zip_code_type="STANDARD" ) - print("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: # get_or_create gives an error if multiple such locations exist (which can happen) - # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) + county = zips[0].get("county") loc = Location.objects.filter( - location_type="CITY_ONLY", city=city, state=state + location_type="FULL_ADDRESS", zipcode=location, city=city, county=county, state=state ) if not loc: loc = Location.objects.create( - location_type="CITY_ONLY", city=city, state=state + location_type="FULL_ADDRESS", zipcode=location, city=city, county=county, state=state ) - print("City " + city + " created") else: loc = loc.first() - print("City " + city + " found") - else: - print( - "No zipcodes found corresponding to city " + city + ", " + state - ) - raise Exception( - "No zipcodes found corresponding to city " + city + ", " + state - ) + msg = "No zipcodes found corresponding to city " + city + ", " + state + raise Exception(msg) self._check_geography_unique(community, geography_type, city) elif geography_type == "COUNTY": @@ -350,10 +321,8 @@ def _update_locations(self, geography_type, locations, community): zips = zipcodes.filter_by( county=county, state=state, zip_code_type="STANDARD" ) - print("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: # get_or_create gives an error if multiple such locations exist (which can happen) - # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) loc = Location.objects.filter( location_type="COUNTY_ONLY", county=county, state=state ) @@ -361,24 +330,12 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="COUNTY_ONLY", county=county, state=state ) - print("County " + county + " created") else: loc = loc.first() - print("County " + county + " found") else: - print( - "No zipcodes found corresponding to county " - + county - + ", " - + state - ) - raise Exception( - "No zipcodes found corresponding to county " - + county - + ", " - + state - ) + msg = "No zipcodes found corresponding to county " + county + ", " + state + raise Exception(msg) self._check_geography_unique(community, geography_type, county) @@ -397,15 +354,11 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="STATE_ONLY", state=state ) - print("State " + state + " created") else: loc = loc.first() - print("State " + state + " found") else: - print("No zipcodes found corresponding to state " + location) - raise Exception( - "No zipcodes found corresponding to state " + location - ) + msg = "No zipcodes found corresponding to state " + location + raise Exception(msg) self._check_geography_unique(community, geography_type, location) @@ -413,7 +366,6 @@ def _update_locations(self, geography_type, locations, community): # check that this state is found in the zipcodes list country = location zips = zipcodes.filter_by(country=country, zip_code_type="STANDARD") - print("Number of zipcodes = " + str(len(zips))) if len(zips) > 0: # get_or_create gives an error if multiple such locations exist (which can happen) # loc, created = Location.objects.get_or_create(location_type='ZIP_CODE_ONLY', zipcode=location, city=city) @@ -424,15 +376,11 @@ def _update_locations(self, geography_type, locations, community): loc = Location.objects.create( location_type="COUNTRY_ONLY", country=country ) - print("Country " + country + " created") else: loc = loc.first() - print("Country " + country + " found") else: - print("No zipcodes found corresponding to country " + location) - raise Exception( - "No zipcodes found corresponding to country " + location - ) + msg = "No zipcodes found corresponding to country " + location + raise Exception(msg) self._check_geography_unique(community, geography_type, location) diff --git a/src/api/store/deviceprofile.py b/src/api/store/deviceprofile.py index c8dbb82f0..9fcbdd497 100644 --- a/src/api/store/deviceprofile.py +++ b/src/api/store/deviceprofile.py @@ -2,7 +2,6 @@ from database.models import UserProfile, DeviceProfile, Location, Community from _main_.utils.massenergize_errors import MassEnergizeAPIError, InvalidResourceError, CustomMassenergizeError from _main_.utils.context import Context -from _main_.settings import DEBUG from _main_.utils.massenergize_logger import log from typing import Tuple diff --git a/src/api/store/download.py b/src/api/store/download.py index 46729508b..49f408cfc 100644 --- a/src/api/store/download.py +++ b/src/api/store/download.py @@ -5,7 +5,6 @@ InvalidResourceError, CustomMassenergizeError, ) -from _main_.utils.massenergize_response import MassenergizeResponse from _main_.utils.context import Context from collections import Counter from api.store.utils import get_human_readable_date @@ -44,18 +43,15 @@ from api.constants import STANDARD_USER, GUEST_USER from _main_.utils.constants import ADMIN_URL_ROOT, COMMUNITY_URL_ROOT from api.store.tag_collection import TagCollectionStore -from api.store.deviceprofile import DeviceStore from django.db.models import Q from _main_.utils.massenergize_logger import log from typing import Tuple from django.utils import timezone import datetime -from django.utils.timezone import utc -from carbon_calculator.carbonCalculator import AverageImpact from django.db.models import Count, Sum from uuid import UUID - +from carbon_calculator.carbonCalculator import getCarbonImpact EMPTY_DOWNLOAD = (None, None) @@ -448,13 +444,8 @@ def _get_last_30_days_count(self, action): #Gets row information for the All Actions CSV and the All Communities and Actions CSV def _get_action_info_cells(self, action): - average_carbon_points = ( - AverageImpact(action.calculator_action) - if action.calculator_action - else int(action.average_carbon_score) - if action.average_carbon_score.isdigit() - else 0 - ) + + average_carbon_points = getCarbonImpact(action) is_published = "Yes" if action.is_published else "No" @@ -575,10 +566,8 @@ def _get_team_info_cells(self, team): done_actions_count += actions.filter(status= "DONE").count() for done_action in done_actions: - if done_action.action and done_action.action.calculator_action: - total_carbon_points += ( - AverageImpact(done_action.action.calculator_action, done_action.date_completed) - ) + total_carbon_points += getCarbonImpact(done_action) + total_carbon_points = str(total_carbon_points) trending_actions = self._get_last_30_days_list(members) @@ -738,9 +727,7 @@ def _get_community_info_cells(self, community, prim_comm_dict): carbon_user_reported = sum( [ - AverageImpact(action_rel.action.calculator_action) - if action_rel.action.calculator_action - else 0 + getCarbonImpact(action_rel) for action_rel in done_action_rels ] ) diff --git a/src/api/store/graph.py b/src/api/store/graph.py index 77cb673c4..4c67be16b 100644 --- a/src/api/store/graph.py +++ b/src/api/store/graph.py @@ -9,7 +9,7 @@ from typing import Tuple from api.services.utils import send_slack_message from _main_.settings import SLACK_SUPER_ADMINS_WEBHOOK_URL, RUN_SERVER_LOCALLY, IS_PROD, IS_CANARY -from carbon_calculator.carbonCalculator import AverageImpact +from carbon_calculator.carbonCalculator import getCarbonImpact def get_households_engaged(community: Community): @@ -24,7 +24,7 @@ def get_households_engaged(community: Community): # loop over actions completed for actionRel in done_actions: if actionRel.action and actionRel.action.calculator_action : - points = AverageImpact(actionRel.action.calculator_action, actionRel.date_completed) + points = getCarbonImpact(actionRel) carbon_footprint_reduction += points return {"community": {"id": community.id, "name": community.name}, @@ -38,9 +38,8 @@ def get_all_households_engaged(): actions_completed = done_actions.count() carbon_footprint_reduction = 0 for actionRel in done_actions: - if actionRel.action and actionRel.action.calculator_action : - carbon_footprint_reduction += AverageImpact(actionRel.action.calculator_action, actionRel.date_completed) - + carbon_footprint_reduction += getCarbonImpact(actionRel) + return {"community": {"id": 0, "name": 'Other'}, "actions_completed": actions_completed, "households_engaged": households_engaged, "carbon_footprint_reduction": carbon_footprint_reduction} diff --git a/src/api/store/misc.py b/src/api/store/misc.py index d49d80574..96c9e75da 100644 --- a/src/api/store/misc.py +++ b/src/api/store/misc.py @@ -1,22 +1,31 @@ -from typing import Tuple - -from _main_.utils.massenergize_logger import log - -from _main_.utils.context import Context +""" + Miscellaneous routes: store layer +""" from _main_.utils.footage.spy import Spy +from api.tests.common import createUsers +from database.models import ( + Action, + Vendor, + Event, + Menu, + UserProfile, + HomePageSettings, +) + Community, from _main_.utils.massenergize_errors import ( CustomMassenergizeError, InvalidResourceError, MassEnergizeAPIError, ) +from _main_.utils.context import Context +from database.models import CarbonEquivalency +from .utils import get_community +from typing import Tuple +from api.utils.api_utils import get_viable_menu_items from _main_.utils.utils import load_json from api.tests.common import createUsers from api.utils.api_utils import load_default_menus_from_json, \ remove_unpublished_menu_items, validate_menu_content -from database.models import Action, CarbonEquivalency, Community, CommunityAdminGroup, CommunityMember, Data, Event, \ - FeatureFlag, HomePageSettings, Location, Media, Menu, RealEstateUnit, Subdomain, TagCollection, Team, TeamMember, \ - UserActionRel, \ - UserProfile, Vendor from database.utils.common import json_loader from .utils import check_location, find_reu_community, get_community, split_location_string from ..constants import MENU_CONTROL_FEATURE_FLAGS @@ -93,303 +102,6 @@ def actions_report(self, context: Context, args) -> Tuple[list, MassEnergizeAPIE print("Total actions = "+str(total) + ", no CCAction ="+str(total_wo_ccaction)) return None, None - def backfill(self, context: Context, args) -> Tuple[list, MassEnergizeAPIError]: - return self.backfill_graph_default_data(context, args), None - - def backfill_subdomans(self): - for c in Community.objects.all(): - try: - Subdomain(name=c.subdomain, in_use=True, community=c).save() - except Exception as e: - print(e) - - def backfill_teams( - self, context: Context, args - ) -> Tuple[list, MassEnergizeAPIError]: - try: - teams = Team.objects.all() - for team in teams: - members = team.members.all() - for member in members: - team_member = TeamMember.objects.filter( - user=member, team=team - ).first() - if team_member: - team_member.is_admin = False - team_member.save() - if not team_member: - team_member = TeamMember.objects.create( - user=member, team=team, is_admin=False - ) - - admins = team.admins.all() - for admin in admins: - team_member = TeamMember.objects.filter( - user=admin, team=team - ).first() - if team_member: - team_member.is_admin = True - team_member.save() - else: - team_member = TeamMember.objects.create( - user=admin, team=team, is_admin=True - ) - - return {"teams_member_backfill": "done"}, None - except Exception as e: - log.exception(e) - return None, CustomMassenergizeError(e) - - def backfill_community_members( - self, context: Context, args - ) -> Tuple[list, MassEnergizeAPIError]: - try: - #users = UserProfile.objects.all() - users = UserProfile.objects.filter(is_deleted=False) - for user in users: - for community in user.communities.all(): - community_member: CommunityMember = CommunityMember.objects.filter( - community=community, user=user - ).first() - - if community_member: - community_member.is_admin = False - community_member.save() - else: - community_member = CommunityMember.objects.create( - community=community, user=user, is_admin=False - ) - - admin_groups = CommunityAdminGroup.objects.all() - for group in admin_groups: - for member in group.members.all(): - community_member: CommunityMember = CommunityMember.objects.filter( - community=group.community, user=member - ).first() - if community_member: - community_member.is_admin = True - community_member.save() - else: - community_member = CommunityMember.objects.create( - community=group.community, user=member, is_admin=True - ) - - return {"name": "community_member_backfill", "status": "done"}, None - except Exception as e: - log.exception(e) - return None, CustomMassenergizeError(e) - - def backfill_graph_default_data(self, context: Context, args): - try: - for community in Community.objects.all(): - for tag in TagCollection.objects.get( - name__icontains="Category" - ).tag_set.all(): - d = Data.objects.filter(community=community, name=tag.name).first() - if d: - oldval = d.value - val = 0 - - if community.is_geographically_focused: - user_actions = UserActionRel.objects.filter( - real_estate_unit__community=community, status="DONE" - ) - else: - user_actions = UserActionRel.objects.filter( - action__community=community, status="DONE" - ) - for user_action in user_actions: - if ( - user_action.action - and user_action.action.tags.filter(pk=tag.id).exists() - ): - val += 1 - - d.value = val - d.save() - print( - "Backfill: Community: " - + community.name - + ", Category: " - + tag.name - + ", Old: " - + str(oldval) - + ", New: " - + str(val) - ) - return {"graph_default_data": "done"}, None - - except Exception as e: - log.exception(e) - return None, CustomMassenergizeError(e) - - def backfill_real_estate_units(self, context: Context, args): - ZIPCODE_FIXES = json_loader("api/store/ZIPCODE_FIXES.json") - try: - # BHN - Feb 2021 - assign all real estate units to geographic communities - # Set the community of a real estate unit based on the location of the real estate unit. - # This defines what geographic community, if any gets credit - # For now, check for zip code - reu_all = RealEstateUnit.objects.all() - print("Number of real estate units:" + str(reu_all.count())) - - userProfiles = UserProfile.objects.prefetch_related( - "real_estate_units" - ).filter(is_deleted=False) - print("number of user profiles:" + str(userProfiles.count())) - - # loop over profiles and realEstateUnits associated with them - for userProfile in userProfiles: - user = userProfile.email - reus = userProfile.real_estate_units.all() - msg = "User: %s (%s), %d households - %s" % ( - userProfile.full_name, - userProfile.email, - reus.count(), - userProfile.created_at.strftime("%Y-%m-%d"), - ) - print(msg) - - for reu in reus: - street = unit_number = city = county = state = zip = "" - loc = reu.location # a JSON field - zip = None - - if loc: - # if not isinstance(loc,str): - # # one odd case in dev DB, looked like a Dict - # print("REU location not a string: "+str(loc)+" Type="+str(type(loc))) - # loc = loc["street"] - - loc_parts = split_location_string(loc) - if len(loc_parts) >= 4: - # deal with odd cases - if userProfile.email in ZIPCODE_FIXES: - zip = ZIPCODE_FIXES[user]["zipcode"] - city = ZIPCODE_FIXES[user]["city"] - else: - street = loc_parts[0].capitalize() - city = loc_parts[1].capitalize() - state = loc_parts[2].upper() - zip = loc_parts[3].strip() - if not zip or (len(zip) != 5 and len(zip) != 10): - print( - "Invalid zipcode: " + zip + ", setting to 00000" - ) - zip = "00000" - elif len(zip) == 10: - zip = zip[0:5] - else: - # deal with odd cases which were encountered in the dev database - zip = "00000" - state = "MA" # may be wrong occasionally - for entry in ZIPCODE_FIXES: - if loc.find(entry) >= 0: - zip = ZIPCODE_FIXES[entry]["zipcode"] - city = ZIPCODE_FIXES[entry]["city"] - state = ZIPCODE_FIXES[entry].get("state", "MA") - - print("Zipcode assigned " + zip) - - # create the Location for the RealEstateUnit - location_type, valid = check_location( - street, unit_number, city, state, zip - ) - if not valid: - print("check_location returns: " + location_type) - continue - - # newloc, created = Location.objects.get_or_create( - newloc = Location.objects.filter( - location_type=location_type, - street=street, - unit_number=unit_number, - zipcode=zip, - city=city, - county=county, - state=state, - ) - if not newloc: - newloc = Location.objects.create( - location_type=location_type, - street=street, - unit_number=unit_number, - zipcode=zip, - city=city, - county=county, - state=state, - ) - print("Zipcode " + zip + " created for town " + city) - else: - newloc = newloc.first() - print("Zipcode " + zip + " found for town " + city) - - reu.address = newloc - reu.save() - - else: - - # fixes for some missing addresses in summer Prod DB - zip = "00000" - cn = "" - if userProfile.communities: - cn = userProfile.communities.first().name - elif reu.community: - cn = reu.community.name - - if cn in ZIPCODE_FIXES: - zip = ZIPCODE_FIXES[cn]["zipcode"] - city = ZIPCODE_FIXES[cn]["city"] - elif user in ZIPCODE_FIXES: - zip = ZIPCODE_FIXES[user]["zipcode"] - city = ZIPCODE_FIXES[user]["city"] - - # no location was stored? - if zip == "00000": - print("No location found for RealEstateUnit " + str(reu)) - - location_type = "ZIP_CODE_ONLY" - newloc, created = Location.objects.get_or_create( - location_type=location_type, - street=street, - unit_number=unit_number, - zipcode=zip, - city=city, - county=county, - state=state, - ) - if created: - print("Location with zipcode " + zip + " created") - else: - print("Location with zipcode " + zip + " found") - reu.address = newloc - reu.save() - - # determine which, if any, community this household is actually in - community = find_reu_community(reu) - if community: - print( - "Adding the REU with zipcode " - + zip - + " to the community " - + community.name - ) - reu.community = community - - elif reu.community: - print( - "REU not located in any community, but was labeled as belonging to the community " - + reu.community.name - ) - reu.community = None - reu.save() - - return {"backfill_real_estate_units": "done"}, None - - except Exception as e: - log.exception(e) - return None, CustomMassenergizeError(e) - def create_carbon_equivalency(self, args): try: new_carbon_equivalency = CarbonEquivalency.objects.create(**args) diff --git a/src/api/store/team.py b/src/api/store/team.py index 46fba3c45..3f9769c3b 100644 --- a/src/api/store/team.py +++ b/src/api/store/team.py @@ -9,10 +9,9 @@ from _main_.utils.massenergize_errors import MassEnergizeAPIError, InvalidResourceError, CustomMassenergizeError, NotAuthorizedError from _main_.utils.context import Context from _main_.utils.constants import COMMUNITY_URL_ROOT, ADMIN_URL_ROOT -from .utils import get_community_or_die, get_user_or_die, get_admin_communities, getCarbonScoreFromActionRel, unique_media_filename -from _main_.utils.massenergize_logger import log +from .utils import get_community_or_die, get_user_or_die, get_admin_communities, unique_media_filename +from carbon_calculator.carbonCalculator import getCarbonImpact from _main_.utils.emailer.send_email import send_massenergize_email, send_massenergize_email_with_attachments -from carbon_calculator.carbonCalculator import AverageImpact from typing import Tuple from django.db.models import Q def can_set_parent(parent, this_team=None): @@ -117,7 +116,7 @@ def team_stats(self, context: Context, args) -> Tuple[list, MassEnergizeAPIError res["actions_todo"] += actions.filter(status="TODO").count() for done_action in done_actions: if done_action.action and done_action.action.calculator_action: - res["carbon_footprint_reduction"] += AverageImpact(done_action.action.calculator_action, done_action.date_completed) + res["carbon_footprint_reduction"] += getCarbonImpact(done_action) ans.append(res) @@ -638,7 +637,7 @@ def list_actions_completed(self, context: Context, args) -> Tuple[list, MassEner completed_actions = UserActionRel.objects.filter(user__in=users, is_deleted=False).select_related('action', 'action__calculator_action') for completed_action in completed_actions: action_id = completed_action.action.id - action_carbon = getCarbonScoreFromActionRel(completed_action) + action_carbon = getCarbonImpact(completed_action) done = 1 if completed_action.status == "DONE" else 0 todo = 1 if completed_action.status == "TODO" else 0 diff --git a/src/api/store/userprofile.py b/src/api/store/userprofile.py index 7f0cccd31..ec98d6a74 100644 --- a/src/api/store/userprofile.py +++ b/src/api/store/userprofile.py @@ -11,14 +11,13 @@ from _main_.utils.massenergize_errors import MassEnergizeAPIError, InvalidResourceError, CustomMassenergizeError, NotAuthorizedError from _main_.utils.massenergize_response import MassenergizeResponse from _main_.utils.context import Context -from _main_.settings import DEBUG, IS_PROD, IS_CANARY +from _main_.settings import DEBUG, IS_PROD, IS_CANARY, SLACK_SUPER_ADMINS_WEBHOOK_URL from _main_.utils.massenergize_logger import log from .utils import get_community, get_user_from_context, get_user_or_die, get_community_or_die, get_admin_communities, remove_dups, \ find_reu_community, split_location_string, check_location import json from typing import Tuple from api.services.utils import send_slack_message -from _main_.settings import SLACK_SUPER_ADMINS_WEBHOOK_URL, IS_PROD, IS_CANARY, DEBUG from _main_.utils.constants import COMMUNITY_URL_ROOT, ME_LOGO_PNG from api.utils.constants import GUEST_USER_EMAIL_TEMPLATE, ME_SUPPORT_TEAM_EMAIL, MOU_SIGNED_ADMIN_RECIPIENT, MOU_SIGNED_SUPPORT_TEAM_TEMPLATE from _main_.utils.emailer.send_email import send_massenergize_email, send_massenergize_email_with_attachments @@ -33,7 +32,7 @@ def remove_locked_fields(args): return args -def _get_or_create_reu_location(args, user=None): +def _get_or_create_reu_location(args): unit_type = args.pop('unit_type', None) location = args.pop('location', None) @@ -77,10 +76,6 @@ def _get_or_create_reu_location(args, user=None): country=country ) - if created: - print("Location with zipcode " , zipcode , " created for user " , user.preferred_name) - else: - print("Location with zipcode " , zipcode , " found for user " , user.preferred_name) return reuloc def _update_action_data_totals(action, household, delta): @@ -430,7 +425,7 @@ def add_household(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIEr name = args.pop('name', None) unit_type = args.pop('unit_type', None) - reuloc = _get_or_create_reu_location(args, user) + reuloc = _get_or_create_reu_location(args) reu = RealEstateUnit.objects.create(name=name, unit_type=unit_type) reu.address = reuloc @@ -453,14 +448,13 @@ def edit_household(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIE if not user: return None, CustomMassenergizeError("sign_in_required / provide user_id or user_email") name = args.pop('name', None) - unit_type = args.pop('unit_type', None) + household_id = args.get('household_id', None) if not household_id: return None, CustomMassenergizeError("Please provide household_id") - reuloc = _get_or_create_reu_location(args, user) - - + reuloc = _get_or_create_reu_location(args) + reu = RealEstateUnit.objects.get(pk=household_id) reu.name = name reu.unit_type = args.get("unit_type", "RESIDENTIAL") @@ -586,8 +580,6 @@ def create_user(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIErro else: user_info['user_type'] = new_user_type - # allow home address to be passed in - location = args.pop('location', '') profile_picture = args.pop("profile_picture", None) color = args.pop('color', '') @@ -675,8 +667,13 @@ def create_user(self, context: Context, args) -> Tuple[dict, MassEnergizeAPIErro # create their first household, if a location was specified, and if they don't have a household reu = user.real_estate_units.all() if reu.count() == 0: - household = RealEstateUnit.objects.create(name="Home", unit_type="residential", community=community, - location=location) + + # RealEstateUnit should have address, not location + reuloc = _get_or_create_reu_location(args) + household = RealEstateUnit.objects.create(name="Home", unit_type="residential", community=community) + household.address = reuloc + household.save() + user.real_estate_units.add(household) user.save() diff --git a/src/api/store/utils.py b/src/api/store/utils.py index e0f9be42b..f1f5f3a98 100644 --- a/src/api/store/utils.py +++ b/src/api/store/utils.py @@ -1,26 +1,16 @@ from _main_.settings import IS_LOCAL, IS_PROD, IS_CANARY from _main_.utils.metrics import timed from _main_.utils.utils import strip_website -from database.models import Community, UserProfile, RealEstateUnit, Location, CustomCommunityWebsiteDomain -from _main_.utils.massenergize_errors import CustomMassenergizeError, InvalidResourceError +from database.models import Community, UserProfile, CustomCommunityWebsiteDomain +from _main_.utils.massenergize_errors import CustomMassenergizeError from _main_.utils.context import Context from django.db.models import Q from django.core.files.uploadedfile import InMemoryUploadedFile from database.utils.constants import SHORT_STR_LEN import zipcodes import datetime -from carbon_calculator.carbonCalculator import AverageImpact from _main_.utils.massenergize_logger import log - -def getCarbonScoreFromActionRel(actionRel): - if not actionRel or actionRel.status !="DONE": return 0 - if actionRel.carbon_impact : return actionRel.carbon_impact - calculator_action = actionRel.action.calculator_action - if calculator_action: - return AverageImpact(calculator_action, actionRel.date_completed) - return 0 - def get_community(community_id=None, subdomain=None): try: if community_id: diff --git a/src/carbon_calculator/CCDefaults.py b/src/carbon_calculator/CCDefaults.py index 2c8b48abf..6302cc29e 100644 --- a/src/carbon_calculator/CCDefaults.py +++ b/src/carbon_calculator/CCDefaults.py @@ -36,8 +36,8 @@ def date_import(date_string): date = datetime.strptime(date_string, '%Y-%m-%d') return date.date() -def getDefault(locality, variable, date=None, default=None): - return CCD.getDefault(CCD,locality, variable, date, default=default) +def getDefault(loc_options, variable, date=None, default=None): + return CCD.getDefault(CCD, loc_options, variable, date, default=default) def removeDuplicates(): # assuming which duplicate is removed doesn't matter... @@ -90,29 +90,37 @@ def loadDefaults(self): print(str(e)) print("CalcDefault initialization skipped") - def getDefault(self, locality, variable, date, default=None): + def getDefault(self, loc_options, variable, date, default=None): # load default values if they haven't yet been loaded if self.DefaultsByLocality["default"]=={}: self.loadDefaults(self) - if locality not in self.DefaultsByLocality: - locality = "default" - if variable in self.DefaultsByLocality[locality]: - # variable found; get the value appropriate for the date - var = self.DefaultsByLocality[locality][variable] # not a copy - if date==None: - # if date not specified, use the most recent value - value = var["values"][-1] - else: - for i in range(len(var["valid_dates"])): - valid_date = var["valid_dates"][i] - if valid_date < date: - value = var["values"][i] - return value - - # no defaults found. Signal this as an error. + # eliminate any location options which aren't tracked + options = [] + for locality in loc_options: + if locality in self.DefaultsByLocality: + options.append(locality) + # default is the standard option + options.append("default") + + for locality in options: + if variable in self.DefaultsByLocality[locality]: + # variable found; get the value appropriate for the date + var = self.DefaultsByLocality[locality][variable] # not a copy + if date==None: + # if date not specified, use the most recent value + value = var["values"][-1] + else: + for i in range(len(var["valid_dates"])): + valid_date = var["valid_dates"][i] + if valid_date < date: + value = var["values"][i] + return value + + # if a default value was specified, return it if default: return default + # no defaults specified. Signal this as an error. raise Exception('Carbon Calculator error: value for "'+variable+'" not found in CalcDefaults') def exportDefaults(self,fileName): diff --git a/src/carbon_calculator/carbonCalculator.py b/src/carbon_calculator/carbonCalculator.py index 34d4145e3..3d1171152 100644 --- a/src/carbon_calculator/carbonCalculator.py +++ b/src/carbon_calculator/carbonCalculator.py @@ -119,12 +119,69 @@ def SavePic2Media(picURL): print("Error encountered: "+str(e)) return None -def AverageImpact(action, date=None, locality="default"): + +def getCarbonImpact(action, done_only=True): + + if not action: return 0 + + if hasattr(action, "status"): + # this is a UserActionRel for a completed or todo action + if done_only and action.status !="DONE": return 0 + + household = action.real_estate_unit + if household.address: + loc_options = locality_options(action.real_estate_unit.address.simple_json()) + else: + if household.community: + location = household.community.locations.first().simple_json() + loc_options = locality_options(location) + else: + loc_options = [] + if action.action and action.action.calculator_action: + return AverageImpact(action.action.calculator_action, action.date_completed, loc_options) + else: + return action.carbon_impact + + elif hasattr(action, "calculator_action"): + # This is an Action posted by the community. In this case we use the community location (community.locations) + location = action.community.locations.first().simple_json() + loc_options = locality_options(location) + if action.calculator_action: + return AverageImpact(action.calculator_action, None, loc_options) + else: + return 0 + else: + return 0 + +def AverageImpact(action, date=None, loc_options=[]): averageName = action.name + '_average_points' - impact = getDefault(locality, averageName, date, default=TOKEN_POINTS) + impact = getDefault(loc_options, averageName, date, default=TOKEN_POINTS) return impact + +def locality_options(loc): + # for actions which have been completed, use RealEstateUnit location + # for actions posted by communities, but not done we use the Community location + # return options in precedence order: city, county, state, i.e. ["Concord-MA", "Middlesex County-MA", "MA"] + options = [] + if type(loc) == dict: + city = loc.get("city") + county = loc.get("county") + state = loc.get("state") + if city and state: + options.append(city + "-" + state) + if county: + options.append(county + "-" + state) + options.append(state) + return options + # not usual: + # invalid location information, use default + print("carbon calculator: Locality type = "+str(type(loc))+ " loc = "+str(loc)) + return [] + + class CarbonCalculator: + def __init__(self, reset=False) : start = time.time() @@ -202,6 +259,7 @@ def __init__(self, reset=False) : print(str(e)) print("Calculator initialization skipped") + # query actions def Query(self,action=None): if action in self.allActions: diff --git a/src/carbon_calculator/models.py b/src/carbon_calculator/models.py index 2df7056bd..ad2f0eccc 100644 --- a/src/carbon_calculator/models.py +++ b/src/carbon_calculator/models.py @@ -1,7 +1,7 @@ from django.db import models from django.forms.models import model_to_dict from database.utils.constants import SHORT_STR_LEN, TINY_STR_LEN -from database.utils.common import json_loader, get_json_if_not_none +from database.utils.common import json_loader # Create your models here. @@ -207,167 +207,7 @@ def __str__(self): class Meta: db_table = 'questions_cc' -#class Station(models.Model): -# id = models.AutoField(primary_key=True) -# name = models.CharField(max_length=NAME_STR_LEN, unique=True) -# displayname = models.CharField(max_length=NAME_STR_LEN,blank=True) -# description = models.CharField(max_length=SHORT_STR_LEN) -# icon = models.ForeignKey(CarbonCalculatorMedia, on_delete=models.SET_NULL, null=True, related_name='cc_station_icon') -# actions = models.JSONField(blank=True, null=True) -# -# def simple_json(self): -# return model_to_dict(self) -# -# def full_json(self): -# return self.simple_json() -# -# def __str__(self): -# return self.displayname -# -# class Meta: -# db_table = 'stations_cc' -# -# -# -#class Group(models.Model): -# from database.models import UserProfile -# id = models.AutoField(primary_key=True) -# name = models.CharField(max_length=NAME_STR_LEN,unique=True) -# displayname = models.CharField(max_length=NAME_STR_LEN,blank=True) -# description = models.CharField(max_length=SHORT_STR_LEN, blank=True) -# member_list = models.ManyToManyField(UserProfile, related_name='group_members', blank=True) -# members = models.PositiveIntegerField(default=0) -# points = models.PositiveIntegerField(default=0) -# savings = models.DecimalField(default=0.0,max_digits=10,decimal_places=2) -# -# def simple_json(self): -# return model_to_dict(self) -# -# def full_json(self): -# return self.simple_json() -# -# def __str__(self): -# return self.displayname -# -# class Meta: -# db_table = 'groups_cc' -# -#class Org(models.Model): -# """ -# A class used to represent an organization -# -# Attributes -# ---------- -# email : str -# email of the user. Should be unique. -# created_at: DateTime -# The date and time that this goal was added -# created_at: DateTime -# The date and time of the last time any updates were made to the information -# about this goal -# -# """ -# name = models.CharField(max_length=SHORT_STR_LEN,blank=True) -# contact = models.CharField(max_length=SHORT_STR_LEN,blank=True) -# email = models.EmailField() -# phone = models.CharField(max_length=TINY_STR_LEN, blank=True) -# about = models.CharField(max_length=LONG_STR_LEN, blank=True) -# url = models.URLField(blank=True) -# logo = models.ForeignKey(CarbonCalculatorMedia,on_delete=models.SET_NULL, -# null=True, blank=True, related_name='event_host_logo') -# -# def simple_json(self): -# return { -# "name": self.name, -# "contact": self.contact, -# "email": self.email, -# "phone": self.phone, -# "about": self.about, -# "url":self.url, -# "logo":get_json_if_not_none(self.logo) -# } -# -# -# def full_json(self): -# return self.simple_json() -# -# def __str__(self): -# return self.name -# -# class Meta: -# db_table = 'organization_cc' -# -#class Event(models.Model): -# from database.models import UserProfile -# id = models.AutoField(primary_key=True) -# name = models.CharField(max_length=NAME_STR_LEN,unique=True) -# displayname = models.CharField(max_length=NAME_STR_LEN,blank=True) -# datetime = models.DateTimeField(blank=True, null=True) -# location = models.CharField(max_length=SHORT_STR_LEN,blank=True) -## stations = models.ForeignKey(Station, on_delete=models.SET_NULL, -## null=True, blank=True, related_name='cc_station_picture') -# stationslist = models.JSONField(null=True, blank=True) -# groups = models.ManyToManyField(Group,blank=True) -# host_org = models.ManyToManyField(Org,blank=True,related_name='host_orgs') -# sponsor_org = models.ManyToManyField(Org,blank=True,related_name='sponsor_orgs') -## updated 4/24/20 -## for a given event, campaign or purpose (platform default or community sites) -# visible = models.BooleanField(default=True) -# event_tag = models.CharField(max_length=TINY_STR_LEN,blank=True) -# #attendees = models.ManyToManyField(CalcUser, blank=True) -# attendees = models.ManyToManyField(UserProfile, blank=True) -# -# def simple_json(self): -# return model_to_dict(self) -# -# def full_json(self): -# return self.simple_json() -# -# def __str__(self): -# return self.displayname -# -# class Meta: -# db_table = 'events_cc' - -#class ActionPoints(models.Model): -# """ -# Class to record choices made for actions - first from the Event Calculator and eventually from -# """ -# from database.models import UserProfile -# id = models.AutoField(primary_key=True) -# #user = models.ForeignKey(CalcUser, blank=True, null=True, on_delete=models.SET_NULL) -# user = models.ForeignKey(UserProfile, blank=True, null=True, on_delete=models.SET_NULL) -# -# created_date = models.DateTimeField(auto_now_add=True) -## -# #action = models.ForeignKey(Action, blank=True, null=True, on_delete=models.SET_NULL) -# action = models.CharField(max_length=NAME_STR_LEN, blank=True) -# -## the questions and answers -# choices = models.JSONField(blank=True) -# -# #next two fields added 11/15/20 -# action_date = models.DateTimeField(auto_now_add=True, null=True) -# # 'pledged', 'todo', 'done' -# action_status = models.CharField(max_length=NAME_STR_LEN,blank=True) -## -# points = models.IntegerField(default = 0) -# cost = models.IntegerField(default = 0) -# savings = models.IntegerField(default = 0) -# -# def simple_json(self): -# return model_to_dict(self) -# -# def full_json(self): -# return self.simple_json() -## -# def __str__(self): -# return "%s-%s-(%s)" % (self.action, self.user, self.created_date) -# -# class Meta: -# db_table = 'action_points_cc' -# -# + class CalcDefault(models.Model): """ Class to keep track of calculator assumptions by locality diff --git a/src/database/models.py b/src/database/models.py index 2a66e1f29..c5ef4fb62 100644 --- a/src/database/models.py +++ b/src/database/models.py @@ -9,17 +9,18 @@ from _main_.utils.base_model import BaseModel from _main_.utils.base_model import RootModel from apps__campaigns.helpers import get_user_accounts -from database.utils.settings.model_constants.events import EventConstants from django.db import models from django.db.models.fields import BooleanField from _main_.utils.feature_flags.FeatureFlagConstants import FeatureFlagConstants from _main_.utils.footage.FootageConstants import FootageConstants from database.utils.constants import * +from database.utils.settings.model_constants.events import EventConstants from database.utils.settings.admin_settings import AdminPortalSettings from database.utils.settings.model_constants.user_media_uploads import ( UserMediaConstants, ) from database.utils.settings.user_settings import UserPortalSettings +from carbon_calculator.carbonCalculator import getCarbonImpact from django.utils import timezone from django.core.files.storage import default_storage from django.db.models.query import QuerySet @@ -34,7 +35,6 @@ from api.constants import COMMUNITY_NOTIFICATION_TYPES, STANDARD_USER, GUEST_USER from django.forms.models import model_to_dict from carbon_calculator.models import Action as CCAction -from carbon_calculator.carbonCalculator import AverageImpact CHOICES = json_loader("./database/raw_data/other/databaseFieldChoices.json") ZIP_CODE_AND_STATES = json_loader("./database/raw_data/other/states.json") @@ -669,10 +669,7 @@ def full_json(self): carbon_footprint_reduction = 0 for actionRel in done_actions: - if actionRel.action and actionRel.action.calculator_action: - carbon_footprint_reduction += AverageImpact( - actionRel.action.calculator_action, actionRel.date_completed - ) + carbon_footprint_reduction += getCarbonImpact(actionRel) goal["organic_attained_carbon_footprint_reduction"] = carbon_footprint_reduction @@ -1026,20 +1023,14 @@ def summary(self): ).prefetch_related("action__calculator_action") done_points = 0 for actionRel in done_actions: - if actionRel.action and actionRel.action.calculator_action: - done_points += AverageImpact(actionRel.action.calculator_action, actionRel.date_completed) - else: - done_points += actionRel.carbon_impact + done_points += getCarbonImpact(actionRel) todo_actions = UserActionRel.objects.filter( user=self, status="TODO" ).prefetch_related("action__calculator_action") todo_points = 0 for actionRel in todo_actions: - if actionRel.action and actionRel.action.calculator_action: - todo_points += AverageImpact(actionRel.action.calculator_action, actionRel.date_completed) - else: - todo_points += actionRel.carbon_impact + todo_points += getCarbonImpact(actionRel, False) # second arg for TODO actions user_testimonials = Testimonial.objects.filter( is_deleted=False, is_approved=True, user=self @@ -2128,7 +2119,7 @@ def full_json(self): "name": u.real_estate_unit.name if u.real_estate_unit else None, }, "date_completed": u.date_completed, - "carbon_impact": AverageImpact(u.action.calculator_action, u.date_completed) if u.action.calculator_action else None, + "carbon_impact": getCarbonImpact(u), "recorded_at": u.updated_at, } for u in UserActionRel.objects.filter(action=self, is_deleted=False) diff --git a/src/database/utils/common.py b/src/database/utils/common.py index c752f5380..5d171e1d8 100644 --- a/src/database/utils/common.py +++ b/src/database/utils/common.py @@ -2,17 +2,12 @@ This file contains utility functions that come in handy for processing and retrieving data """ -from ast import Lambda import base64 import hashlib import json -from django.core import serializers -from django.forms.models import model_to_dict from collections.abc import Iterable from _main_.settings import AWS_S3_REGION_NAME, AWS_STORAGE_BUCKET_NAME from _main_.utils.massenergize_logger import log - -from _main_.utils.utils import Console import boto3 s3 = boto3.client('s3', region_name=AWS_S3_REGION_NAME) diff --git a/src/task_queue/database_tasks/backfill_data.py b/src/task_queue/database_tasks/backfill_data.py new file mode 100644 index 000000000..1652c34ab --- /dev/null +++ b/src/task_queue/database_tasks/backfill_data.py @@ -0,0 +1,342 @@ +""" +This File contains tasks used to fix missing or incorrect data in certain database models. +These were formerly in a 'data.backfill' route in misc.py. +Some are obsolete and not relevant any longer, but kept for posterity in case needed. + +1. backfill_subdomains: loop over communities, created Subdomain instances for each +2. backfill_teams: loop through Teams, create TeamMember instances +3. backfill_community_memebers: loop through UserProfile and CommunityAdminGroups to create CommunityJembers +4. backfill_graph_default_data: fix action done category data +5. backfill_locations: add more complete address fields to RealEstateUnits and Communities +""" +import csv +import datetime +from _main_.utils.massenergize_logger import log +import zipcodes +from django.http import HttpResponse +from database.models import ( + Subdomain, + Community, + Team, + TeamMember, + CommunityMember, + RealEstateUnit, + CommunityAdminGroup, + UserProfile, + Data, + TagCollection, + UserActionRel, + Location, +) +from database.utils.common import json_loader +from _main_.utils.emailer.send_email import send_massenergize_email_with_attachments +from api.store.utils import find_reu_community, split_location_string, check_location +from api.utils.constants import DATA_DOWNLOAD_TEMPLATE + + +def write_to_csv(data): + response = HttpResponse(content_type="text/csv") + writer = csv.DictWriter(response, fieldnames=["Message", "Error"]) + writer.writeheader() + for row in data: + writer.writerow(row) + return response.content + + +def backfill_data(task=None): + try: + #data = backfill_subdomains() + #data = backfill_teams() + #data = backfill_community_members() + #data = backfill_graph_default_data() + data = backfill_locations() + if len(data) > 0: + report = write_to_csv(data) + temp_data = {'data_type': "Content Spacing", "name":task.creator.full_name if task.creator else "admin"} + file_name = "Data-Backfill-Report-{}.csv".format(datetime.datetime.now().strftime("%Y-%m-%d")) + send_massenergize_email_with_attachments(DATA_DOWNLOAD_TEMPLATE,temp_data,[task.creator.email], report, file_name) + return True + + except Exception as e: + log.exception(e) + return False + + +def backfill_subdomans(): + data = [] + for c in Community.objects.all(): + try: + data.append({"Message": "Creating "+c.subdomain, "Error":""}) + Subdomain(name=c.subdomain, in_use=True, community=c).save() + except Exception as e: + data.append({"Message": "Error", "Error": str(e)}) + return data + + +def backfill_teams(): + data = [] + try: + teams = Team.objects.all() + for team in teams: + members = team.members.all() + for member in members: + team_member = TeamMember.objects.filter( + user=member, team=team + ).first() + if team_member: + team_member.is_admin = False + team_member.save() + if not team_member: + team_member = TeamMember.objects.create( + user=member, team=team, is_admin=False + ) + admins = team.admins.all() + for admin in admins: + team_member = TeamMember.objects.filter( + user=admin, team=team + ).first() + if team_member: + team_member.is_admin = True + team_member.save() + else: + team_member = TeamMember.objects.create( + user=admin, team=team, is_admin=True + ) + return data + + except Exception as e: + log.exception(e) + data.append({"Message": "Fatal error", "Error": str(e)}) + return data + +def backfill_community_members(): + data = [] + try: + users = UserProfile.objects.filter(is_deleted=False) + for user in users: + for community in user.communities.all(): + community_member: CommunityMember = CommunityMember.objects.filter( + community=community, user=user + ).first() + if community_member: + community_member.is_admin = False + community_member.save() + else: + community_member = CommunityMember.objects.create( + community=community, user=user, is_admin=False + ) + admin_groups = CommunityAdminGroup.objects.all() + for group in admin_groups: + for member in group.members.all(): + community_member: CommunityMember = CommunityMember.objects.filter( + community=group.community, user=member + ).first() + if community_member: + community_member.is_admin = True + community_member.save() + else: + community_member = CommunityMember.objects.create( + community=group.community, user=member, is_admin=True + ) + return data + except Exception as e: + log.exception(e) + data.append({"Message": "Fatal error", "Error": str(e)}) + return data + + +def backfill_graph_default_data(): + data = [] + try: + for community in Community.objects.all(): + for tag in TagCollection.objects.get( + name__icontains="Category" + ).tag_set.all(): + d = Data.objects.filter(community=community, name=tag.name).first() + if d: + oldval = d.value + val = 0 + if community.is_geographically_focused: + user_actions = UserActionRel.objects.filter( + real_estate_unit__community=community, status="DONE" + ) + else: + user_actions = UserActionRel.objects.filter( + action__community=community, status="DONE" + ) + for user_action in user_actions: + if ( + user_action.action + and user_action.action.tags.filter(pk=tag.id).exists() + ): + val += 1 + d.value = val + d.save() + message = "Community: %s, Category: %s, Old: %s, New: %x" % ( + community.name, tag.name, str(oldval), str(val)) + data.append({"Message": message, "Error":""}) + + return data + except Exception as e: + log.exception(e) + data.append({"Message": "Fatal error", "Error": str(e)}) + return data + + +def backfill_locations(): + CHOICES = json_loader("./database/raw_data/other/databaseFieldChoices.json") + location_types = CHOICES.get("LOCATION_TYPES") + + data = [] + # these were some bad addresses, already fixed + # ZIPCODE_FIXES = json_loader("api/store/ZIPCODE_FIXES.json") + try: + # Update Community locations to be not just a zip code + data.append({"Message": "Updating community locations", "Error":""}) + communities = Community.objects.filter(is_deleted=False) + for community in communities: + # a community can have multiple locations (for example zip codes) + for location in community.locations.all(): + location_type = location.location_type + msg = "Community: %s, location type: %s" % (community.name, location_type) + data.append({"Message": msg, "Error":""}) + if location_type == "ZIP_CODE_ONLY": + # usual case + if location.zipcode and location.zipcode.isnumeric(): + loc_data = zipcodes.matching(location.zipcode)[0] + else: + # training community - a problem + loc_data = zipcodes.filter_by(city=location.city, state=location.state) + if not loc_data: + data.append({"Message": "Skipping", "Error":"Incomplete location"}) + continue + loc_data = loc_data[0] + + data.append({"Message": "Updating to FULL_ADDRESS", "Error":""}) + location.city = loc_data["city"] + location.county = loc_data["county"] + location.state = loc_data["state"] + location.country = loc_data["country"] + location.location_type = "FULL_ADDRESS" + location.save() + + elif location_type == "CITY_ONLY" and (not location.county or not location.country): + # add the county if missing + data.append({"Message": "Updating with county", "Error":""}) + loc_data = zipcodes.filter_by( + city=location.city, state=location.state, zip_code_type="STANDARD" + )[0] + location.county = loc_data["county"] + location.country = loc_data["country"] + location.save() + + # BHN - Feb 2021 - assign all real estate units to geographic communities + # Set the community of a real estate unit based on the location of the real estate unit. + # This defines what geographic community, if any gets credit + # For now, check for zip code + data.append({"Message": "", "Error":""}) + data.append({"Message": "Updating real estate unit locations", "Error":""}) + reu_all = RealEstateUnit.objects.filter(is_deleted=False) + msg = "Number of real estate units:" + str(reu_all.count()) + data.append({"Message": msg, "Error":""}) + + userProfiles = UserProfile.objects.prefetch_related( + "real_estate_units" + ).filter(is_deleted=False) + msg = "number of user profiles:" + str(userProfiles.count()) + data.append({"Message": msg, "Error":""}) + + # loop over profiles and realEstateUnits associated with them + for userProfile in userProfiles: + reus = userProfile.real_estate_units.all() + msg = "User: %s (%s), %d households - %s" % ( + userProfile.full_name, + userProfile.email, + reus.count(), + userProfile.created_at.strftime("%Y-%m-%d"), + ) + data.append({"Message": msg, "Error":""}) + + for reu in reus: + street = unit_number = city = county = state = zip = "" + loc = reu.location # a JSON field + zip = None + if loc: + # if not isinstance(loc,str): + # # one odd case in dev DB, looked like a Dict + # print("REU location not a string: "+str(loc)+" Type="+str(type(loc))) + # loc = loc["street"] + loc_parts = split_location_string(loc) + if len(loc_parts) >= 4: + # deal with odd cases + street = loc_parts[0].capitalize() + city = loc_parts[1].capitalize() + state = loc_parts[2].upper() + zip = loc_parts[3].strip() + if not zip or (len(zip) != 5 and len(zip) != 10): + msg = "Invalid zipcode: " + zip + ", setting to 00000" + data.append({"Message": "Warning", "Error":msg}) + zip = "00000" + elif len(zip) == 10: + zip = zip[0:5] + else: + # deal with odd cases which were encountered in the dev database + zip = "00000" + state = "MA" # may be wrong occasionally + msg = "Zipcode assigned " + zip + data.append({"Message": msg, "Error":""}) + + # create the Location for the RealEstateUnit + try: + if zipcodes.is_real(zip): + info = zipcodes.matching(zip)[0] + city = info["city"] + state = info["state"] + county = info["county"] + country = info["country"] + location_type, valid = check_location( + street, unit_number, city, state, zip + ) + if not valid: + msg = "check_location returns: " + location_type + data.append({"Message": "Warning", "Error":msg}) + continue + + msg = "Updating location for REU ID ", str(reu.id) + data.append({"Message": msg, "Error":""}) + + newloc, created = Location.objects.get_or_create( + location_type=location_type, + street=street, + unit_number=unit_number, + zipcode=zip, + city=city, + county=county, + state=state, + ) + if created: + msg = "Zipcode " + zip + " created for town " + city + else: + msg = "Zipcode " + zip + " found for town " + city + data.append({"Message": msg, "Error":""}) + reu.address = newloc + reu.save() + + community = find_reu_community(reu) + if community: + msg = "Adding the REU with zipcode "+ zip+ " to the community "+ community.name + data.append({"Message": msg, "Error":""}) + reu.community = community + elif reu.community: + msg = "REU not located in any community, but was labeled as belonging to the community "+ reu.community.name + data.append({"Message": msg, "Error":""}) + reu.community = None + reu.save() + except Exception as e: + data.append({"Message": "Error", "Error": str(e)}) + return data + + except Exception as e: + log.exception(e) + data.append({"Message": "Fatal error", "Error": str(e)}) + return data diff --git a/src/task_queue/helpers.py b/src/task_queue/helpers.py index 6dc0eee08..63e17a25a 100644 --- a/src/task_queue/helpers.py +++ b/src/task_queue/helpers.py @@ -19,7 +19,10 @@ def is_time_to_run(task): if last_run is None: return True - if freq == ONE_OFF and not last_run == today: + + # One-offs can be run multiple times in a day + five_minutes_ago = today - relativedelta(minutes=5) + if freq == ONE_OFF and not last_run == five_minutes_ago: return True if freq == DAILY: diff --git a/src/task_queue/jobs.py b/src/task_queue/jobs.py index d33427ca1..5cc70cc34 100644 --- a/src/task_queue/jobs.py +++ b/src/task_queue/jobs.py @@ -2,6 +2,7 @@ from task_queue.database_tasks.contents_spacing_correction import process_spacing_data from task_queue.database_tasks.translate_db_content import TranslateDBContents from task_queue.database_tasks.update_actions_content import update_actions_content +from task_queue.database_tasks.backfill_data import backfill_data from task_queue.nudges.cadmin_events_nudge import send_events_nudge from task_queue.nudges.user_event_nudge import prepare_user_events_nudge from task_queue.nudges.postmark_sender_signature import collect_and_create_signatures @@ -28,4 +29,5 @@ "Update Action Content": update_actions_content, "Remove Duplicate Images": remove_duplicate_images, "Translate Database Contents": TranslateDBContents().start_translations, + "Backfill Database Models": backfill_data } \ No newline at end of file diff --git a/src/task_queue/tasks.py b/src/task_queue/tasks.py index 5c11e934f..5dfd7d1e7 100644 --- a/src/task_queue/tasks.py +++ b/src/task_queue/tasks.py @@ -1,5 +1,4 @@ import datetime -import logging from celery import shared_task from django.db import transaction diff --git a/src/task_queue/views.py b/src/task_queue/views.py index 087426d58..6ec996763 100644 --- a/src/task_queue/views.py +++ b/src/task_queue/views.py @@ -10,14 +10,14 @@ YEARLY_MOU_TEMPLATE, ) from api.constants import STANDARD_USER, GUEST_USER -from database.models import FeatureFlag, UserProfile, UserActionRel, Community, CommunityAdminGroup, CommunityMember, Event, RealEstateUnit, Team, Testimonial, Vendor, PolicyConstants, PolicyAcceptanceRecords, CommunitySnapshot, Goal, Action +from database.models import UserProfile, UserActionRel, Community, CommunityAdminGroup, CommunityMember, Event, RealEstateUnit, Team, Testimonial, Vendor, PolicyConstants, PolicyAcceptanceRecords, CommunitySnapshot, Goal, Action +from carbon_calculator.carbonCalculator import getCarbonImpact from django.utils import timezone import datetime from django.utils.timezone import utc from django.db.models import Count from django.db.models import Q from django.core.exceptions import ObjectDoesNotExist -from carbon_calculator.carbonCalculator import AverageImpact today = parse_datetime_to_aware() one_week_ago = today - timezone.timedelta(days=7) @@ -254,9 +254,7 @@ def _get_user_reported_info(community, users): carbon_user_reported = sum( [ - AverageImpact(action_rel.action.calculator_action, action_rel.date_completed) - if action_rel.action.calculator_action - else 0 + getCarbonImpact(action_rel) for action_rel in done_action_rels ] )