Modified the ExperimentalMimeParser to record raw boundary markers

MimeParser and previous iterations of the ExperimentalMimeParser did not
record the exact boundary marker used in the MIME stream to separate each
child of a multipart.

Normally, this isn't a problem because the boundary marker used by properly
written email software is just "--{boundary}\r\n", BUT according to the
MIME specifications, these lines are allowed to contain trailing whitespace
characters that should be ignored by MIME parsers (which is what MimeParser
and ExperimentalMimeParser have been doing).

That said, when writing a Multipart back out to a stream, if we do not
record the *exact* boundary markers used between each child part, then we
cannot write out an *exact* copy of what the parser encountered in the
original MIME stream.

As far as I have been able to tell, this is an extremely uncommon scenario
but in the interest of persuing perfection, ExperimentalMimeParser and
Multipart have now been modified to handle this.

MimeParser still suffers from this flaw, but I'll be looking into solving
that next.

Author: jstedfast

MimeKit/AsyncMimeParser.cs

@@ -482,8 +482,11 @@ async Task ConstructMultipartAsync (Multipart multipart, MimeEntityEndEventArgs
 				//OnMultipartEndBoundaryBegin (multipart, GetEndOffset (inputIndex));
 
 				// consume the end boundary and read the epilogue (if there is one)
-				multipart.WriteEndBoundary = true;
 				await SkipLineAsync (false, cancellationToken).ConfigureAwait (false);
+
+				// FIXME: we should save the raw end boundary marker in case it contains trailing whitespace
+				multipart.RawEndBoundary = null;
+
 				PopBoundary ();
 
 				//OnMultipartEndBoundaryEnd (multipart, GetOffset (inputIndex));
@@ -498,8 +501,6 @@ async Task ConstructMultipartAsync (Multipart multipart, MimeEntityEndEventArgs
 			endOffset = GetEndOffset (inputIndex);
 			args.Lines = GetLineCount (beginLineNumber, beginOffset, endOffset);
 
-			multipart.WriteEndBoundary = false;
-
 			// We either found the end of the stream or we found a parent's boundary
 			PopBoundary ();
 		}

MimeKit/ExperimentalMimeParser.cs

@@ -57,6 +57,8 @@ public class ExperimentalMimeParser : MimeReader, IMimeParser
 		byte[] preHeaderBuffer;
 		int preHeaderLength;
 
+		byte[] rawBoundary;
+
 		// MimePart content and Multipart preamble/epilogue state
 		Stream content;
 
@@ -251,17 +253,26 @@ public void SetStream (Stream stream, bool persistent)
 			SetStream (stream, MimeFormat.Default, persistent);
 		}
 
-		void PopEntity ()
+		void PushEntity (MimeEntity entity)
 		{
-			if (stack.Count > 1) {
-				var entity = (MimeEntity) stack.Pop ();
+			if (stack.Count > 0) {
 				var parent = stack.Peek ();
 
-				if (parent is MimeMessage message)
+				if (parent is Multipart multipart) {
+					multipart.AddInternal (entity, rawBoundary);
+					rawBoundary = null;
+				} else if (parent is MimeMessage message) {
 					message.Body = entity;
-				else
-					((Multipart) parent).InternalAdd (entity);
+				}
 			}
+
+			stack.Push (entity);
+		}
+
+		void PopEntity ()
+		{
+			if (stack.Count > 1)
+				stack.Pop ();
 		}
 
 		#region Mbox Events
@@ -390,6 +401,12 @@ protected override void OnMimeMessageBegin (long beginOffset, int beginLineNumbe
 				Buffer.BlockCopy (preHeaderBuffer, 0, message.MboxMarker, 0, preHeaderLength);
 			}
 
+			if (stack.Count > 0) {
+				var rfc822 = (MessagePart) stack.Peek ();
+
+				rfc822.Message = message;
+			}
+
 			stack.Push (message);
 		}
 
@@ -408,12 +425,8 @@ protected override void OnMimeMessageBegin (long beginOffset, int beginLineNumbe
 		/// <param name="cancellationToken">The cancellation token.</param>
 		protected override void OnMimeMessageEnd (long beginOffset, int beginLineNumber, long headersEndOffset, long endOffset, int lines, CancellationToken cancellationToken)
 		{
-			if (stack.Count > 1) {
-				var message = (MimeMessage) stack.Pop ();
-				var rfc822 = (MessagePart) stack.Peek ();
-
-				rfc822.Message = message;
-			}
+			if (stack.Count > 1)
+				stack.Pop ();
 		}
 
 		#endregion MimeMessage Events
@@ -436,7 +449,7 @@ protected override void OnMimePartBegin (ContentType contentType, long beginOffs
 			var toplevel = stack.Count > 0 && stack.Peek () is MimeMessage;
 			var part = Options.CreateEntity (contentType, headers, toplevel, depth);
 
-			stack.Push (part);
+			PushEntity (part);
 		}
 
 		/// <summary>
@@ -545,7 +558,7 @@ protected override void OnMessagePartBegin (ContentType contentType, long beginO
 			var rfc822 = Options.CreateEntity (contentType, headers, toplevel, depth);
 
 			parsingMessageHeaders = true;
-			stack.Push (rfc822);
+			PushEntity (rfc822);
 			depth++;
 		}
 
@@ -589,27 +602,10 @@ protected override void OnMultipartBegin (ContentType contentType, long beginOff
 			var toplevel = stack.Count > 0 && stack.Peek () is MimeMessage;
 			var multipart = Options.CreateEntity (contentType, headers, toplevel, depth);
 
-			stack.Push (multipart);
+			PushEntity (multipart);
 			depth++;
 		}
 
-		/// <summary>
-		/// Called when a multipart end boundary is encountered in the stream.
-		/// </summary>
-		/// <remarks>
-		/// Called when a multipart end boundary is encountered in the stream.
-		/// </remarks>
-		/// <param name="beginOffset">The offset into the stream where the boundary marker began.</param>
-		/// <param name="lineNumber">The line number where the boundary marker was found in the stream.</param>
-		/// <param name="endOffset">The offset into the stream where the boundary marker ended.</param>
-		/// <param name="cancellationToken">The cancellation token.</param>
-		protected override void OnMultipartEndBoundaryEnd (long beginOffset, int lineNumber, long endOffset, CancellationToken cancellationToken)
-		{
-			var multipart = (Multipart) stack.Peek ();
-
-			multipart.WriteEndBoundary = true;
-		}
-
 		/// <summary>
 		/// Called when the beginning of the preamble of a multipart is encountered in the stream.
 		/// </summary>
@@ -663,6 +659,47 @@ protected override void OnMultipartPreambleEnd (long beginOffset, int beginLineN
 			content = null;
 		}
 
+		/// <summary>
+		/// Called when a multipart boundary is encountered in the stream.
+		/// </summary>
+		/// <remarks>
+		/// Called when a multipart boundary is encountered in the stream.
+		/// </remarks>
+		/// <param name="buffer">The buffer containing the boundary marker.</param>
+		/// <param name="startIndex">The index denoting the starting position of the boundary marker within the buffer.</param>
+		/// <param name="count">The length of the boundary marker within the buffer, in bytes.</param>
+		/// <param name="beginOffset">The offset into the stream where the boundary marker began.</param>
+		/// <param name="lineNumber">The line number where the boundary marker exists within the stream.</param>
+		/// <param name="cancellationToken">The cancellation token.</param>
+		protected override void OnMultipartBoundaryRead (byte[] buffer, int startIndex, int count, long beginOffset, int lineNumber, CancellationToken cancellationToken)
+		{
+			rawBoundary = new byte[count];
+
+			Buffer.BlockCopy (buffer, startIndex, rawBoundary, 0, count);
+		}
+
+		/// <summary>
+		/// Called when a multipart end boundary is encountered in the stream.
+		/// </summary>
+		/// <remarks>
+		/// Called when a multipart end boundary is encountered in the stream.
+		/// </remarks>
+		/// <param name="buffer">The buffer containing the boundary marker.</param>
+		/// <param name="startIndex">The index denoting the starting position of the boundary marker within the buffer.</param>
+		/// <param name="count">The length of the boundary marker within the buffer, in bytes.</param>
+		/// <param name="beginOffset">The offset into the stream where the boundary marker began.</param>
+		/// <param name="lineNumber">The line number where the boundary marker exists within the stream.</param>
+		/// <param name="cancellationToken">The cancellation token.</param>
+		protected override void OnMultipartEndBoundaryRead (byte[] buffer, int startIndex, int count, long beginOffset, int lineNumber, CancellationToken cancellationToken)
+		{
+			var multipart = (Multipart) stack.Peek ();
+			var rawEndBoundary = new byte[count];
+
+			Buffer.BlockCopy (buffer, startIndex, rawEndBoundary, 0, count);
+
+			multipart.RawEndBoundary = rawEndBoundary;
+		}
+
 		/// <summary>
 		/// Called when the beginning of the epilogue of a multipart is encountered in the stream.
 		/// </summary>

MimeKit/MimeParser.cs

@@ -1720,8 +1720,11 @@ unsafe void ConstructMultipart (Multipart multipart, MimeEntityEndEventArgs args
 				//OnMultipartEndBoundaryBegin (multipart, GetEndOffset (inputIndex));
 
 				// consume the end boundary and read the epilogue (if there is one)
-				multipart.WriteEndBoundary = true;
 				SkipLine (inbuf, false, cancellationToken);
+
+				// FIXME: we should save the raw end boundary marker in case it contains trailing whitespace
+				multipart.RawEndBoundary = null;
+
 				PopBoundary ();
 
 				//OnMultipartEndBoundaryEnd (multipart, GetOffset (inputIndex));
@@ -1736,8 +1739,6 @@ unsafe void ConstructMultipart (Multipart multipart, MimeEntityEndEventArgs args
 			endOffset = GetEndOffset (inputIndex);
 			args.Lines = GetLineCount (beginLineNumber, beginOffset, endOffset);
 
-			multipart.WriteEndBoundary = false;
-
 			// We either found the end of the stream or we found a parent's boundary
 			PopBoundary ();
 		}