Feature/custom user model

example_project/main/models.py

@@ -1,6 +1,7 @@
 import logging
 
-from django.contrib.auth.models import User
+from django.contrib.auth import get_user_model
+User = get_user_model()
 # from django.core.mail import get_connection
 from django.core.mail import EmailMessage
 # from django.db import models

example_project/settings.py

@@ -173,7 +173,7 @@
 SENDGRID_EMAIL_PORT = 587
 SENDGRID_EMAIL_USERNAME = os.getenv("SENDGRID_EMAIL_USERNAME")
 SENDGRID_EMAIL_PASSWORD = os.getenv("SENDGRID_EMAIL_PASSWORD")
-# SENDGRID_CREATE_MISSING_EMAIL_MESSAGES = True
+# SENDGRID_CREATE_EVENTS_AND_EMAILS_FOR_NEWSLETTERS = True
 
 try:
 	from settings_local import *

sendgrid/constants.py

@@ -4,6 +4,9 @@
 ARGUMENT_DATA_TYPE_FLOAT = 3
 ARGUMENT_DATA_TYPE_COMPLEX = 4
 ARGUMENT_DATA_TYPE_STRING = 5
+
+BATCHED_EVENT_SEPARATOR = "\r\n"
+
 EVENT_SHORT_DESC_MAX_LENGTH = 32
 
 EVENT_FIELDS = ("event","category","email")
@@ -34,8 +37,31 @@
 	"SPAMREPORT": (),
 }
 
-UNIQUE_ARGS_STORED_FOR_EVENTS_WITHOUT_MESSAGE_ID = (
+NEWSLETTER_UNIQUE_IDENTIFIER = "newsletter[newsletter_id]"
+UNIQUE_ARGS_STORED_FOR_NEWSLETTER_EVENTS = (
 	"newsletter[newsletter_id]",
 	"newsletter[newsletter_send_id]",
 	"newsletter[newsletter_user_list_id]",
-)
+)
+
+TEST_SENDER_EMAIL = "ryan@example.com"
+TEST_RECIPIENTS = ["ryan@example.com", "tom@example.com","anotherguy@example.com"]
+
+SAMPLE_NEWSLETTER_IDS = {
+	"newsletter_send_id": "952852", 
+	"newsletter_id": "916273", 
+	"newsletter_user_list_id": "5059777"
+}
+
+SAMPLE_NEWSLETTER_IDS_2 = {
+	"newsletter_send_id": "666", 
+	"newsletter_id": "4324324", 
+	"newsletter_user_list_id": "2344324"
+}
+
+SAMPLE_EVENT_DICT_WITHOUT_MESSAGE_ID_OR_TIMESTAMP = {
+	"email": TEST_RECIPIENTS[0],
+	"category":["category1"],
+	"event": "OPEN"
+}
+

sendgrid/mixins.py

@@ -1,4 +1,5 @@
 from django.conf import settings
+from django.db import models, transaction, IntegrityError
 from django.utils import simplejson
 
 from utils import add_unsubscribes
@@ -9,6 +10,43 @@
 SENDGRID_EMAIL_USERNAME = getattr(settings, "SENDGRID_EMAIL_USERNAME", None)
 SENDGRID_EMAIL_PASSWORD = getattr(settings, "SENDGRID_EMAIL_PASSWORD", None)
 
+class BulkCreateManager(models.Manager):
+	@transaction.commit_on_success
+	def bulk_create_with_post_save(self,instances):
+		instancesCreated = self.bulk_create(instances)
+		for instance in instancesCreated:
+			#this should possibly be abandoned for a custom bulk_post_save signal for efficiency reasons
+			models.signals.post_save.send(
+				sender=instance.__class__, 
+				instance=instance,
+				created=True, 
+				raw=False,
+				using='default'
+			)
+		return instancesCreated
+
+	def bulk_create_with_manual_ids(self,instances):
+		try:
+			start = self.select_for_update().all().order_by('-pk')[0].pk + 1
+		except IndexError:
+			start = 1
+		for i,instance in enumerate(instances): 
+			instance.pk = start + i
+
+		return self.bulk_create_with_post_save(instances)
+
+	def bulk_create_with_manual_ids_retry(self,instances,max_retries=5,retry_counter=0):
+		try:
+			return self.bulk_create_with_manual_ids(instances)
+		except IntegrityError, e:
+			if "Duplicate" in e.__str__() and "PRIMARY" in e.__str__() and retry_counter < max_retries:
+				return self.bulk_create_with_manual_ids_retry(
+					instances=instances,
+					max_retries=max_retries,
+					retry_counter=retry_counter+1
+				)
+			else:
+				raise e
 
 class SendGridUserMixin:
 	"""
@@ -36,4 +74,4 @@ def delete_from_unsubscribes(self):
 		"""
 		response = delete_unsubscribes(email=self.email)
 		result = simplejson.loads(response)
-		return result
+		return result

sendgrid/models.py

@@ -9,6 +9,7 @@
 from django.dispatch import receiver
 from django.utils.translation import ugettext_lazy as _
 
+from .mixins import BulkCreateManager
 from .signals import sendgrid_email_sent
 from .signals import sendgrid_event_recieved
 from sendgrid.constants import (
@@ -18,9 +19,11 @@
 	ARGUMENT_DATA_TYPE_FLOAT,
 	ARGUMENT_DATA_TYPE_COMPLEX,
 	ARGUMENT_DATA_TYPE_STRING,
-	UNIQUE_ARGS_STORED_FOR_EVENTS_WITHOUT_MESSAGE_ID,
+	UNIQUE_ARGS_STORED_FOR_NEWSLETTER_EVENTS,
+	NEWSLETTER_UNIQUE_IDENTIFIER,
 )
 from sendgrid.signals import sendgrid_email_sent
+from sendgrid.utils.formatutils import get_value_from_dict_using_formdata_key
 
 MAX_CATEGORIES_PER_EMAIL_MESSAGE = 10
 
@@ -50,7 +53,8 @@
 EMAIL_MESSAGE_TO_EMAIL_MAX_LENGTH = 254
 
 if SENDGRID_USER_MIXIN_ENABLED:
-	from django.contrib.auth.models import User
+	from django.contrib.auth import get_user_model
+	User = get_user_model()
 	from .mixins import SendGridUserMixin
 
 	User.__bases__ += (SendGridUserMixin,)
@@ -64,6 +68,7 @@ def update_email_message(sender, message, response, **kwargs):
 	emailMessage.response = response
 	emailMessage.save()
 
+
 def save_email_message(sender, **kwargs):
 	message = kwargs.get("message", None)
 	response = kwargs.get("response", None)
@@ -104,6 +109,8 @@ def save_email_message(sender, **kwargs):
 			response=response,
 		)
 
+		logger.debug("DEBUG-EVENT: emailMessage record Created with message_id:{0}".format(emailMessage.message_id))
+
 		if categories:
 			for categoryName in categories:
 				category, created = Category.objects.get_or_create(name=categoryName)
@@ -141,6 +148,8 @@ def save_email_message(sender, **kwargs):
 				logMessage = "Component {c} is not tracked"
 				logger.debug(logMessage.format(c=component))
 
+	return emailMessage
+
 @receiver(sendgrid_event_recieved)
 def log_event_recieved(sender, request, **kwargs):
 	if settings.DEBUG:
@@ -187,8 +196,8 @@ class Meta:
 	def __unicode__(self):
 		return self.key
 
-
 class EmailMessage(models.Model):
+	objects = BulkCreateManager()
 	message_id = models.CharField(unique=True, max_length=36, editable=False, blank=True, null=True, help_text="UUID")
 	# user = models.ForeignKey(User, null=True) # TODO
 	from_email = models.CharField(max_length=EMAIL_MESSAGE_FROM_EMAIL_MAX_LENGTH, help_text="Sender's e-mail")
@@ -209,12 +218,14 @@ def from_event(self, event_dict):
 		"""
 		Returns a new EmailMessage instance derived from an Event Dictionary.
 		"""
-		newsletter_id = event_dict.get("newsletter[newsletter_id]")
+		newsletter_id = get_value_from_dict_using_formdata_key(NEWSLETTER_UNIQUE_IDENTIFIER,event_dict)
+
 		to_email = event_dict.get("email")
 		try:
-			emailMessage = UniqueArgument.objects.get(data=newsletter_id, argument__key="newsletter[newsletter_id]", email_message__to_email=to_email).email_message
+			emailMessage = UniqueArgument.objects.get(data=newsletter_id, argument__key=NEWSLETTER_UNIQUE_IDENTIFIER, email_message__to_email=to_email).email_message
 		except UniqueArgument.DoesNotExist:
-			categories = [value for key,value in event_dict.items() if 'category' in key]
+			categories = event_dict.get("category",[])
+
 			emailMessageSpec = {
 				"message_id": event_dict.get("message_id", None),
 				"from_email": "",
@@ -223,26 +234,32 @@ def from_event(self, event_dict):
 			}
 			if len(categories) > 0:
 				emailMessageSpec["category"] = categories[0]
-
+
+			if event_dict.get("message_id", None):
+				logger.debug("DEBUG-EVENT: Trying To Create Email From Event with message_id {0}".format(event_dict.get("message_id", None)))
+				existingEmail = EmailMessage.objects.filter(message_id=event_dict.get("message_id", None))
+				if len(existingEmail) > 0:
+					logger.debug("DEBUG-EVENT: Found an existing email with message_id {0}".format(event_dict.get("message_id", None)))
 			emailMessage = EmailMessage.objects.create(**emailMessageSpec)
 
 			for category in categories:
 				categoryObj,created = Category.objects.get_or_create(name=category)
 				emailMessage.categories.add(categoryObj)
 
-			uniqueArgs = {}
-			for key in UNIQUE_ARGS_STORED_FOR_EVENTS_WITHOUT_MESSAGE_ID:
-				uniqueArgs[key] = event_dict.get(key)
+			if newsletter_id:
+				uniqueArgs = {}
+				for key in UNIQUE_ARGS_STORED_FOR_NEWSLETTER_EVENTS:
+					uniqueArgs[key] = get_value_from_dict_using_formdata_key(key,event_dict)
 
-			for argName, argValue in uniqueArgs.items():
-				argument,_ = Argument.objects.get_or_create(
-					key=argName
-				)
-				uniqueArg = UniqueArgument.objects.create(
-					argument=argument,
-					email_message=emailMessage,
-					data=argValue
-				)
+				for argName, argValue in uniqueArgs.items():
+					argument,_ = Argument.objects.get_or_create(
+						key=argName
+					)
+					uniqueArg = UniqueArgument.objects.create(
+						argument=argument,
+						email_message=emailMessage,
+						data=argValue
+					)
 
 		return emailMessage
 
@@ -439,6 +456,7 @@ def __unicode__(self):
 
 
 class Event(models.Model):
+	objects = BulkCreateManager()
 	email_message = models.ForeignKey(EmailMessage)
 	email = models.EmailField()
 	event_type = models.ForeignKey(EventType)

sendgrid/settings.py

@@ -1,4 +1,5 @@
 from django.conf import settings
 
 # This is experimental, use with caution.
-SENDGRID_CREATE_MISSING_EMAIL_MESSAGES = getattr(settings, "SENDGRID_CREATE_MISSING_EMAIL_MESSAGES", False)
+SENDGRID_CREATE_MISSING_EMAILS_FOR_EVENTS_WITH_MESSAGE_ID = getattr(settings, "SENDGRID_CREATE_MISSING_EMAILS_FOR_EVENTS_WITH_MESSAGE_ID", False)
+SENDGRID_CREATE_EVENTS_AND_EMAILS_FOR_NEWSLETTERS = getattr(settings, "SENDGRID_CREATE_EVENTS_AND_EMAILS_FOR_NEWSLETTERS", False)