Modified MimeReader/ExperimentalMimeParser to handle "double boundaries"

Also updated HeaderList to track whether or not it should write a body
separator (i.e. a blank line to separate the headers from the body).

This gets a bunch more ExperimentalMimeParserTests to pass.

Author: jstedfast

MimeKit/AsyncMimeParser.cs

@@ -331,7 +331,7 @@ async Task ConstructMessagePartAsync (MessagePart rfc822, MimeEntityEndEventArgs
 			}
 
 			var type = GetContentType (null);
-			var entity = options.CreateEntity (type, headers, true, depth);
+			var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: true, depth);
 			var entityArgs = new MimeEntityEndEventArgs (entity) {
 				HeadersEndOffset = headerBlockEnd,
 				BeginOffset = headerBlockBegin,
@@ -425,7 +425,7 @@ async Task MultipartScanSubpartsAsync (Multipart multipart, int depth, Cancellat
 				//	return BoundaryType.EndBoundary;
 
 				var type = GetContentType (multipart.ContentType);
-				var entity = options.CreateEntity (type, headers, false, depth);
+				var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: false, depth);
 				var entityArgs = new MimeEntityEndEventArgs (entity, multipart) {
 					HeadersEndOffset = headerBlockEnd,
 					BeginOffset = headerBlockBegin,
@@ -532,7 +532,7 @@ public async Task<HeaderList> ParseHeadersAsync (CancellationToken cancellationT
 
 			state = eos ? MimeParserState.Eos : MimeParserState.Complete;
 
-			var parsed = new HeaderList (options);
+			var parsed = new HeaderList (options, true);
 			foreach (var header in headers)
 				parsed.Add (header);
 
@@ -576,7 +576,7 @@ public async Task<MimeEntity> ParseEntityAsync (CancellationToken cancellationTo
 
 			// Note: we pass 'false' as the 'toplevel' argument here because
 			// we want the entity to consume all the headers.
-			var entity = options.CreateEntity (type, headers, false, 0);
+			var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: false, 0);
 			var entityArgs = new MimeEntityEndEventArgs (entity) {
 				HeadersEndOffset = headerBlockEnd,
 				BeginOffset = headerBlockBegin,
@@ -676,7 +676,7 @@ public async Task<MimeMessage> ParseMessageAsync (CancellationToken cancellation
 			}
 
 			var type = GetContentType (null);
-			var entity = options.CreateEntity (type, headers, true, 0);
+			var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: true, 0);
 			var entityArgs = new MimeEntityEndEventArgs (entity) {
 				HeadersEndOffset = headerBlockEnd,
 				BeginOffset = headerBlockBegin,

MimeKit/AsyncMimeReader.cs

@@ -173,6 +173,7 @@ async Task StepHeadersAsync (CancellationToken cancellationToken)
 
 				// Check for an empty line denoting the end of the header block.
 				if (IsEndOfHeaderBlock (left)) {
+					await OnBodySeparatorAsync (beginOffset, beginLineNumber, GetOffset (inputIndex), CancellationToken.None).ConfigureAwait (false);
 					state = MimeParserState.Content;
 					break;
 				}
@@ -547,21 +548,6 @@ async Task MultipartScanSubpartsAsync (ContentType multipartContentType, int dep
 				state = MimeParserState.Headers;
 				await StepAsync (cancellationToken).ConfigureAwait (false);
 
-				if (state == MimeParserState.Boundary) {
-					if (headerCount == 0) {
-						if (boundary == BoundaryType.ImmediateBoundary)
-							continue;
-
-						return;
-					}
-
-					// This part has no content, but that will be handled in ConstructMultipartAsync()
-					// or ConstructMimePartAsync().
-				}
-
-				//if (state == ParserState.Complete && headers.Count == 0)
-				//	return BoundaryType.EndBoundary;
-
 				var type = GetContentType (multipartContentType);
 				var currentHeadersEndOffset = headerBlockEnd;
 				var currentBeginOffset = headerBlockBegin;

MimeKit/ExperimentalMimeParser.cs

@@ -63,6 +63,7 @@ public class ExperimentalMimeParser : MimeReader, IMimeParser
 		Stream content;
 
 		bool parsingMessageHeaders;
+		bool hasBodySeparator;
 		int depth;
 
 		bool persistent;
@@ -335,6 +336,7 @@ protected override void OnHeadersBegin (long beginOffset, int beginLineNumber, C
 		{
 			headers.Clear ();
 			preHeaderLength = 0;
+			hasBodySeparator = false;
 		}
 
 		/// <summary>
@@ -378,6 +380,22 @@ protected override void OnHeadersEnd (long beginOffset, int beginLineNumber, lon
 			parsingMessageHeaders = false;
 		}
 
+		/// <summary>
+		/// Called when the body separator is encountered in the stream.
+		/// </summary>
+		/// <remarks>
+		/// <para>Called when the body separator is encountered in the stream.</para>
+		/// <para>This method is always called before <see cref="OnHeadersEnd"/> if a body separator is found.</para>
+		/// </remarks>
+		/// <param name="beginOffset">The offset into the stream where the body separator began.</param>
+		/// <param name="lineNumber">The line number where the body separator was found.</param>
+		/// <param name="endOffset">The offset into the stream where the body separator ended.</param>
+		/// <param name="cancellationToken">The cancellation token.</param>
+		protected override void OnBodySeparator (long beginOffset, int lineNumber, long endOffset, CancellationToken cancellationToken)
+		{
+			hasBodySeparator = true;
+		}
+
 		#endregion Header Events
 
 		#region MimeMessage Events
@@ -447,7 +465,7 @@ protected override void OnMimeMessageEnd (long beginOffset, int beginLineNumber,
 		protected override void OnMimePartBegin (ContentType contentType, long beginOffset, int beginLineNumber, CancellationToken cancellationToken)
 		{
 			var toplevel = stack.Count > 0 && stack.Peek () is MimeMessage;
-			var part = Options.CreateEntity (contentType, headers, toplevel, depth);
+			var part = Options.CreateEntity (contentType, headers, hasBodySeparator, toplevel, depth);
 
 			PushEntity (part);
 		}
@@ -555,7 +573,7 @@ protected override void OnMimePartEnd (ContentType contentType, long beginOffset
 		protected override void OnMessagePartBegin (ContentType contentType, long beginOffset, int beginLineNumber, CancellationToken cancellationToken)
 		{
 			var toplevel = stack.Count > 0 && stack.Peek () is MimeMessage;
-			var rfc822 = Options.CreateEntity (contentType, headers, toplevel, depth);
+			var rfc822 = Options.CreateEntity (contentType, headers, hasBodySeparator, toplevel, depth);
 
 			parsingMessageHeaders = true;
 			PushEntity (rfc822);
@@ -600,7 +618,7 @@ protected override void OnMessagePartEnd (ContentType contentType, long beginOff
 		protected override void OnMultipartBegin (ContentType contentType, long beginOffset, int beginLineNumber, CancellationToken cancellationToken)
 		{
 			var toplevel = stack.Count > 0 && stack.Peek () is MimeMessage;
-			var multipart = Options.CreateEntity (contentType, headers, toplevel, depth);
+			var multipart = Options.CreateEntity (contentType, headers, hasBodySeparator, toplevel, depth);
 
 			PushEntity (multipart);
 			depth++;
@@ -838,7 +856,7 @@ public HeaderList ParseHeaders (CancellationToken cancellationToken = default)
 				throw;
 			}
 
-			var parsed = new HeaderList (Options);
+			var parsed = new HeaderList (Options, hasBodySeparator);
 			foreach (var header in headers)
 				parsed.Add (header);
 
@@ -873,7 +891,7 @@ public async Task<HeaderList> ParseHeadersAsync (CancellationToken cancellationT
 				throw;
 			}
 
-			var parsed = new HeaderList (Options);
+			var parsed = new HeaderList (Options, hasBodySeparator);
 			foreach (var header in headers)
 				parsed.Add (header);
 

MimeKit/HeaderList.cs

@@ -50,10 +50,12 @@ public sealed class HeaderList : IList<Header>
 		// this table references the first header of each field
 		readonly Dictionary<string, Header> table;
 		readonly List<Header> headers;
+		bool hasBodySeparator;
 
-		internal HeaderList (ParserOptions options)
+		internal HeaderList (ParserOptions options, bool hasBodySeparator)
 		{
 			table = new Dictionary<string, Header> (MimeUtils.OrdinalIgnoreCase);
+			this.hasBodySeparator = hasBodySeparator;
 			headers = new List<Header> ();
 			Options = options;
 		}
@@ -64,7 +66,7 @@ internal HeaderList (ParserOptions options)
 		/// <remarks>
 		/// Creates a new empty header list.
 		/// </remarks>
-		public HeaderList () : this (ParserOptions.Default.Clone ())
+		public HeaderList () : this (ParserOptions.Default.Clone (), true)
 		{
 		}
 
@@ -704,6 +706,9 @@ public void WriteTo (FormatOptions options, Stream stream, CancellationToken can
 				filtered.Flush (cancellationToken);
 			}
 
+			if (!hasBodySeparator)
+				return;
+
 			if (stream is ICancellableStream cancellable) {
 				cancellable.Write (options.NewLineBytes, 0, options.NewLineBytes.Length, cancellationToken);
 			} else {
@@ -758,6 +763,9 @@ public async Task WriteToAsync (FormatOptions options, Stream stream, Cancellati
 				await filtered.FlushAsync (cancellationToken).ConfigureAwait (false);
 			}
 
+			if (!hasBodySeparator)
+				return;
+
 			await stream.WriteAsync (options.NewLineBytes, 0, options.NewLineBytes.Length, cancellationToken).ConfigureAwait (false);
 		}
 

MimeKit/MimeEntity.cs

@@ -92,7 +92,7 @@ protected MimeEntity (MimeEntityConstructorArgs args)
 			if (args is null)
 				throw new ArgumentNullException (nameof (args));
 
-			Headers = new HeaderList (args.ParserOptions);
+			Headers = new HeaderList (args.ParserOptions, args.HasBodySeparator);
 			ContentType = args.ContentType;
 
 			foreach (var header in args.Headers) {

MimeKit/MimeEntityConstructorArgs.cs

@@ -38,10 +38,12 @@ public sealed class MimeEntityConstructorArgs
 		internal readonly ParserOptions ParserOptions;
 		internal readonly IEnumerable<Header> Headers;
 		internal readonly ContentType ContentType;
+		internal readonly bool HasBodySeparator;
 		internal readonly bool IsTopLevel;
 
-		internal MimeEntityConstructorArgs (ParserOptions options, ContentType ctype, IEnumerable<Header> headers, bool toplevel)
+		internal MimeEntityConstructorArgs (ParserOptions options, ContentType ctype, IEnumerable<Header> headers, bool hasBodySeparator, bool toplevel)
 		{
+			HasBodySeparator = hasBodySeparator;
 			ParserOptions = options;
 			IsTopLevel = toplevel;
 			ContentType = ctype;

MimeKit/MimeMessage.cs

@@ -107,7 +107,7 @@ enum LazyLoadedFields {
 		internal MimeMessage (ParserOptions options, IEnumerable<Header> headers, RfcComplianceMode mode)
 		{
 			addresses = new Dictionary<HeaderId, InternetAddressList> ();
-			Headers = new HeaderList (options);
+			Headers = new HeaderList (options, true);
 
 			compliance = mode;
 
@@ -135,7 +135,7 @@ internal MimeMessage (ParserOptions options, IEnumerable<Header> headers, RfcCom
 		internal MimeMessage (ParserOptions options)
 		{
 			addresses = new Dictionary<HeaderId, InternetAddressList> ();
-			Headers = new HeaderList (options);
+			Headers = new HeaderList (options, true);
 
 			compliance = RfcComplianceMode.Strict;
 
@@ -252,7 +252,7 @@ public MimeMessage (IEnumerable<Header> headers)
 			if (headers is HeaderList headerList) {
 				Headers = headerList;
 			} else {
-				Headers = new HeaderList (ParserOptions.Default.Clone ());
+				Headers = new HeaderList (ParserOptions.Default.Clone (), true);
 
 				foreach (var header in headers)
 					Headers.Add (header);

MimeKit/MimeParser.cs

@@ -1543,7 +1543,7 @@ unsafe void ConstructMessagePart (MessagePart rfc822, MimeEntityEndEventArgs arg
 			}
 
 			var type = GetContentType (null);
-			var entity = options.CreateEntity (type, headers, true, depth);
+			var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: true, depth);
 			var entityArgs = new MimeEntityEndEventArgs (entity) {
 				HeadersEndOffset = headerBlockEnd,
 				BeginOffset = headerBlockBegin,
@@ -1637,7 +1637,7 @@ unsafe void MultipartScanSubparts (Multipart multipart, byte* inbuf, int depth,
 				//	return BoundaryType.EndBoundary;
 
 				var type = GetContentType (multipart.ContentType);
-				var entity = options.CreateEntity (type, headers, false, depth);
+				var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: false, depth);
 				var entityArgs = new MimeEntityEndEventArgs (entity, multipart) {
 					HeadersEndOffset = headerBlockEnd,
 					BeginOffset = headerBlockBegin,
@@ -1779,7 +1779,7 @@ unsafe HeaderList ParseHeaders (byte* inbuf, CancellationToken cancellationToken
 
 			state = eos ? MimeParserState.Eos : MimeParserState.Complete;
 
-			var parsed = new HeaderList (options);
+			var parsed = new HeaderList (options, true);
 			foreach (var header in headers)
 				parsed.Add (header);
 
@@ -1878,7 +1878,7 @@ unsafe MimeEntity ParseEntity (byte* inbuf, CancellationToken cancellationToken)
 
 			// Note: we pass 'false' as the 'toplevel' argument here because
 			// we want the entity to consume all the headers.
-			var entity = options.CreateEntity (type, headers, false, 0);
+			var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: false, 0);
 			var entityArgs = new MimeEntityEndEventArgs (entity) {
 				HeadersEndOffset = headerBlockEnd,
 				BeginOffset = headerBlockBegin,
@@ -1986,7 +1986,7 @@ unsafe MimeMessage ParseMessage (byte* inbuf, CancellationToken cancellationToke
 			}
 
 			var type = GetContentType (null);
-			var entity = options.CreateEntity (type, headers, true, 0);
+			var entity = options.CreateEntity (type, headers, hasBodySeparator: true, toplevel: true, 0);
 			var entityArgs = new MimeEntityEndEventArgs (entity) {
 				HeadersEndOffset = headerBlockEnd,
 				BeginOffset = headerBlockBegin,

MimeKit/MimeReader.cs

@@ -390,6 +390,39 @@ protected virtual Task OnHeadersEndAsync (long beginOffset, int beginLineNumber,
 			return Task.CompletedTask;
 		}
 
+		/// <summary>
+		/// Called when the body separator is encountered in the stream.
+		/// </summary>
+		/// <remarks>
+		/// <para>Called when the body separator is encountered in the stream.</para>
+		/// <para>This method is always called before <see cref="OnHeadersEnd"/> if a body separator is found.</para>
+		/// </remarks>
+		/// <param name="beginOffset">The offset into the stream where the body separator began.</param>
+		/// <param name="lineNumber">The line number where the body separator was found.</param>
+		/// <param name="endOffset">The offset into the stream where the body separator ended.</param>
+		/// <param name="cancellationToken">The cancellation token.</param>
+		protected virtual void OnBodySeparator (long beginOffset, int lineNumber, long endOffset, CancellationToken cancellationToken)
+		{
+		}
+
+		/// <summary>
+		/// Called when the body separator is encountered in the stream.
+		/// </summary>
+		/// <remarks>
+		/// <para>Called when the body separator is encountered in the stream.</para>
+		/// <para>This method is always called before <see cref="OnHeadersEndAsync"/> if a body separator is found.</para>
+		/// </remarks>
+		/// <returns>An asynchronous task context.</returns>
+		/// <param name="beginOffset">The offset into the stream where the body separator began.</param>
+		/// <param name="lineNumber">The line number where the body separator was found.</param>
+		/// <param name="endOffset">The offset into the stream where the body separator ended.</param>
+		/// <param name="cancellationToken">The cancellation token.</param>
+		protected virtual Task OnBodySeparatorAsync (long beginOffset, int lineNumber, long endOffset, CancellationToken cancellationToken)
+		{
+			OnBodySeparator (beginOffset, lineNumber, endOffset, cancellationToken);
+			return Task.CompletedTask;
+		}
+
 		#endregion Header Events
 
 		#region MimeMessage Events
@@ -1860,6 +1893,7 @@ unsafe void StepHeaders (byte* inbuf, CancellationToken cancellationToken)
 
 				// Check for an empty line denoting the end of the header block.
 				if (IsEndOfHeaderBlock (left)) {
+					OnBodySeparator (beginOffset, beginLineNumber, GetOffset (inputIndex), CancellationToken.None);
 					state = MimeParserState.Content;
 					break;
 				}
@@ -2474,21 +2508,6 @@ unsafe void MultipartScanSubparts (ContentType multipartContentType, byte* inbuf
 				state = MimeParserState.Headers;
 				Step (inbuf, cancellationToken);
 
-				if (state == MimeParserState.Boundary) {
-					if (headerCount == 0) {
-						if (boundary == BoundaryType.ImmediateBoundary)
-							continue;
-
-						return;
-					}
-
-					// This part has no content, but that will be handled in ConstructMultipart()
-					// or ConstructMimePart().
-				}
-
-				//if (state == ParserState.Complete && headers.Count == 0)
-				//	return BoundaryType.EndBoundary;
-
 				var type = GetContentType (multipartContentType);
 				var currentHeadersEndOffset = headerBlockEnd;
 				var currentBeginOffset = headerBlockBegin;

MimeKit/ParserOptions.cs

@@ -310,9 +310,9 @@ static bool EqualsAny (string value, params string[] values)
 			return false;
 		}
 
-		internal MimeEntity CreateEntity (ContentType contentType, IList<Header> headers, bool toplevel, int depth)
+		internal MimeEntity CreateEntity (ContentType contentType, IList<Header> headers, bool hasBodySeparator, bool toplevel, int depth)
 		{
-			var args = new MimeEntityConstructorArgs (this, contentType, headers, toplevel);
+			var args = new MimeEntityConstructorArgs (this, contentType, headers, hasBodySeparator, toplevel);
 			var subtype = contentType.MediaSubtype;
 			var type = contentType.MediaType;