修改ReContent为虚方法, 让用户可以扩展读取的方法(编码问题等)

Author: 邹嵩

nuget/DotnetSpider.Core.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
   <metadata>
     <id>DotnetSpider.Core</id>
-    <version>2.4.9</version>
+    <version>2.5.0</version>
     <authors>[email protected];Walterwhatwater;xiaohuan0204</authors>
     <owners>[email protected]</owners>
     <iconUrl>https://github.com/zlzforever/DotnetSpider/blob/master/images/icon.png?raw=true</iconUrl>

nuget/DotnetSpider.Extension.nuspec

@@ -2,7 +2,7 @@
 <package xmlns="http://schemas.microsoft.com/packaging/2012/06/nuspec.xsd">
   <metadata>
     <id>DotnetSpider.Extension</id>
-    <version>2.4.9</version>
+    <version>2.5.0</version>
     <authors>[email protected];Walterwhatwater;xiaohuan0204</authors>
     <owners>[email protected]</owners>
     <iconUrl>https://github.com/zlzforever/DotnetSpider/blob/master/images/icon.png?raw=true</iconUrl>

src/DotnetSpider.Core/Downloader/HttpClientDownloader.cs

@@ -181,6 +181,48 @@ protected override async Task<Page> DowloadContent(Request request, ISpider spid
 			}
 		}
 
+		protected virtual string ReadContent(Site site, HttpResponseMessage response)
+		{
+			byte[] contentBytes = response.Content.ReadAsByteArrayAsync().Result;
+			contentBytes = PreventCutOff(contentBytes);
+			if (string.IsNullOrWhiteSpace(site.EncodingName))
+			{
+				var charSet = response.Content.Headers.ContentType?.CharSet;
+				Encoding htmlCharset = EncodingExtensions.GetEncoding(charSet, contentBytes);
+				return htmlCharset.GetString(contentBytes, 0, contentBytes.Length);
+			}
+			else
+			{
+				return site.Encoding.GetString(contentBytes, 0, contentBytes.Length);
+			}
+		}
+
+		private Page HandleResponse(Request request, HttpResponseMessage response, Site site)
+		{
+			string content = ReadContent(site, response);
+
+			if (_decodeHtml)
+			{
+#if NET45
+				content = HttpUtility.UrlDecode(HttpUtility.HtmlDecode(content), string.IsNullOrEmpty(site.EncodingName) ? Encoding.Default : site.Encoding);
+#else
+				content = System.Net.WebUtility.UrlDecode(System.Net.WebUtility.HtmlDecode(content));
+#endif
+			}
+
+			Page page = new Page(request)
+			{
+				Content = content
+			};
+
+			//foreach (var header in response.Headers)
+			//{
+			//	page.Request.PutExtra(header.Key, header.Value);
+			//}
+
+			return page;
+		}
+
 		private void PrepareHttpClient(HttpClientEntry httpClientEntry)
 		{
 			httpClientEntry.Init(AllowAutoRedirect, () =>
@@ -278,48 +320,6 @@ private HttpRequestMessage GenerateHttpRequestMessage(Request request, Site site
 			return httpRequestMessage;
 		}
 
-		private Page HandleResponse(Request request, HttpResponseMessage response, Site site)
-		{
-			string content = ReadContent(site, response);
-
-			if (_decodeHtml)
-			{
-#if NET45
-				content = HttpUtility.UrlDecode(HttpUtility.HtmlDecode(content), string.IsNullOrEmpty(site.EncodingName) ? Encoding.Default : site.Encoding);
-#else
-				content = System.Net.WebUtility.UrlDecode(System.Net.WebUtility.HtmlDecode(content));
-#endif
-			}
-
-			Page page = new Page(request)
-			{
-				Content = content
-			};
-
-			//foreach (var header in response.Headers)
-			//{
-			//	page.Request.PutExtra(header.Key, header.Value);
-			//}
-
-			return page;
-		}
-
-		private string ReadContent(Site site, HttpResponseMessage response)
-		{
-			byte[] contentBytes = response.Content.ReadAsByteArrayAsync().Result;
-			contentBytes = PreventCutOff(contentBytes);
-			if (string.IsNullOrWhiteSpace(site.EncodingName))
-			{
-				var charSet = response.Content.Headers.ContentType?.CharSet;
-				Encoding htmlCharset = EncodingExtensions.GetEncoding(charSet, contentBytes);
-				return htmlCharset.GetString(contentBytes, 0, contentBytes.Length);
-			}
-			else
-			{
-				return site.Encoding.GetString(contentBytes, 0, contentBytes.Length);
-			}
-		}
-
 		private Page SaveFile(Request request, HttpResponseMessage response, ISpider spider)
 		{
 			var intervalPath = new Uri(request.Url).LocalPath.Replace("//", "/").Replace("/", Env.PathSeperator);