解耦了下载器、数据管道、解析器

Author: zlzforever

Design.zh-CN.md

@@ -0,0 +1,21 @@
+# DESIGN
+
+在此重大更新之前, 下载器 & 选择器 & 数据管理是耦合在一起的, 经过许久思考我决定把它们解耦出来, 这样用户可以自由的选择他们喜爱的组件. 比如说, 下载器使用框架自带的Downloader、WebClientApi、苏菲的HttpHelper等; 解析器可以使用框架自带的Extraction、AngleSharp等;
+
+###  Dowloader
+
+Downloader is a independent module to help user to download data from target website. There are a lot of details, see below:
+
+1. Two ways to set cookie, one is call the AddCookie method in downloader, it add cookie to CookieContainer so impact every request.
+Set cookie header in request, the result is combine you cookie header and cookies in CookieContainer. 
+2. CookieInjector in downloader is invoked one time, and inject cookies to CookieContainer.
+
+### Scheduler
+
+#### Request hash
+
+1. Same url different headers are different requests, so headers are a factor  
+2. There is a CycleRetryTimes property in a request, if value are different, then requests are different. Depth property is not
+a factor.
+
+ 

DotnetSpider.sln

@@ -3,8 +3,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 15
 VisualStudioVersion = 15.0.27703.2042
 MinimumVisualStudioVersion = 10.0.40219.1
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.Common", "src\DotnetSpider.Common\DotnetSpider.Common.csproj", "{F1C6C272-A72A-4A5B-95EE-846643A29A3A}"
-EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.Extraction", "src\DotnetSpider.Extraction\DotnetSpider.Extraction.csproj", "{C5A68E4D-E9B4-4B2D-B198-74FA88C8CA22}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.HtmlAgilityPack.Css", "src\DotnetSpider.HtmlAgilityPack.Css\DotnetSpider.HtmlAgilityPack.Css.csproj", "{38DFF949-761C-4DC1-ADC6-D3F535E84AEF}"
@@ -36,11 +34,12 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		.editorconfig = .editorconfig
 		.gitignore = .gitignore
 		.travis.yml = .travis.yml
+		Design.md = Design.md
+		Design.zh-CN.md = Design.zh-CN.md
 		DistributeDesign.md = DistributeDesign.md
 		publishToNuget.bat = publishToNuget.bat
 		README.md = README.md
 		runtests.sh = runtests.sh
-		Design.md = Design.md
 	EndProjectSection
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.Worker", "src\DotnetSpider.Worker\DotnetSpider.Worker.csproj", "{C416B779-5018-42AF-A1A5-98186389CCED}"
@@ -49,18 +48,14 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.Migrator", "sr
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.Broker.Test", "src\DotnetSpider.Broker.Test\DotnetSpider.Broker.Test.csproj", "{6CAEECB0-0BD0-4A32-B057-99C7DADE3F4C}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotnetSpider.Broker", "src\DotnetSpider.Broker\DotnetSpider.Broker.csproj", "{AAD552D8-0D0A-43B0-9C5D-E542AA8998CE}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "DotnetSpider.Broker", "src\DotnetSpider.Broker\DotnetSpider.Broker.csproj", "{AAD552D8-0D0A-43B0-9C5D-E542AA8998CE}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
 		Release|Any CPU = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(ProjectConfigurationPlatforms) = postSolution
-		{F1C6C272-A72A-4A5B-95EE-846643A29A3A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
-		{F1C6C272-A72A-4A5B-95EE-846643A29A3A}.Debug|Any CPU.Build.0 = Debug|Any CPU
-		{F1C6C272-A72A-4A5B-95EE-846643A29A3A}.Release|Any CPU.ActiveCfg = Release|Any CPU
-		{F1C6C272-A72A-4A5B-95EE-846643A29A3A}.Release|Any CPU.Build.0 = Release|Any CPU
 		{C5A68E4D-E9B4-4B2D-B198-74FA88C8CA22}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
 		{C5A68E4D-E9B4-4B2D-B198-74FA88C8CA22}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{C5A68E4D-E9B4-4B2D-B198-74FA88C8CA22}.Release|Any CPU.ActiveCfg = Release|Any CPU

src/DotnetSpider.Broker.Test/BaseTest.cs

@@ -1,15 +1,10 @@
 using DotnetSpider.Broker.Data;
 using DotnetSpider.Broker.Hubs;
-using Microsoft.AspNetCore.Builder.Internal;
 using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using System;
-using System.Collections.Generic;
-using System.Data;
-using System.Data.SqlClient;
 using System.IO;
-using System.Text;
 using DotnetSpider.Broker.Services;
 
 namespace DotnetSpider.Broker.Test

src/DotnetSpider.Broker.Test/WorkerServiceTest.cs

@@ -1,10 +1,7 @@
 using DotnetSpider.Broker.Data;
-using DotnetSpider.Broker.Hubs;
 using DotnetSpider.Broker.Services;
 using Microsoft.Extensions.DependencyInjection;
 using System;
-using System.Collections.Generic;
-using System.Text;
 using Xunit;
 
 namespace DotnetSpider.Broker.Test

src/DotnetSpider.Broker/ApiAuthorizeMiddleware.cs

@@ -1,7 +1,4 @@
 using Microsoft.AspNetCore.Http;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Net;
 using System.Threading.Tasks;
 

src/DotnetSpider.Broker/BrokerOptions.cs

@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
+using System.Collections.Generic;
 
 namespace DotnetSpider.Broker
 {

src/DotnetSpider.Broker/Controllers/BrokerController.cs

@@ -1,9 +1,5 @@
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Controllers
 {

src/DotnetSpider.Broker/Controllers/HomeController.cs

@@ -1,8 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
-using System.Threading.Tasks;
+using System.Diagnostics;
 using Microsoft.AspNetCore.Mvc;
 using DotnetSpider.Broker.Models;
 

src/DotnetSpider.Broker/Data/Block.cs

@@ -1,7 +1,4 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Data
 {

src/DotnetSpider.Broker/Data/BrokerDbContext.cs

@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Text;
-using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
+using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore;
 
 namespace DotnetSpider.Broker.Data

src/DotnetSpider.Broker/Data/Entity.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections.Generic;
-using System.Linq;
 using System.Reflection;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Data
 {

src/DotnetSpider.Broker/Data/Job.cs

@@ -1,8 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Data
 {

src/DotnetSpider.Broker/Data/JobProperty.cs

@@ -1,8 +1,5 @@
 using System;
-using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Data
 {

src/DotnetSpider.Broker/Data/JobStatus.cs

@@ -1,11 +1,42 @@
-using DotnetSpider.Common;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Converters;
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Data
 {
+	/// <summary>
+	/// 爬虫状态
+	/// </summary>
+	[System.Flags]
+	[JsonConverter(typeof(StringEnumConverter))]
+	public enum Status
+	{
+		/// <summary>
+		/// 初始化
+		/// </summary>
+		Init = 1,
+
+		/// <summary>
+		/// 正在运行
+		/// </summary>
+		Running = 2,
+
+		/// <summary>
+		/// 暂停
+		/// </summary>
+		Paused = 4,
+
+		/// <summary>
+		/// 完成
+		/// </summary>
+		Finished = 8,
+
+		/// <summary>
+		/// 退出
+		/// </summary>
+		Exited = 16
+	}
+
 	public class JobStatus : Entity<Guid>, IHasModificationTime
 	{
 		public virtual Guid Identity { get; set; }

src/DotnetSpider.Broker/Data/NodeStatus.cs

@@ -1,8 +1,5 @@
 using System;
-using System.Collections.Generic;
 using System.ComponentModel.DataAnnotations;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Data
 {

src/DotnetSpider.Broker/Data/Running.cs

@@ -1,7 +1,4 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Data
 {

src/DotnetSpider.Broker/Data/Worker.cs

@@ -1,8 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.ComponentModel.DataAnnotations;
-using System.Linq;
-using System.Threading.Tasks;
+using System.ComponentModel.DataAnnotations;
 
 namespace DotnetSpider.Broker.Data
 {

src/DotnetSpider.Broker/DotnetSpider.Broker.csproj

@@ -17,7 +17,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <ProjectReference Include="..\DotnetSpider.Common\DotnetSpider.Common.csproj" />
     <ProjectReference Include="..\DotnetSpider.Downloader\DotnetSpider.Downloader.csproj" />
   </ItemGroup>
 

src/DotnetSpider.Broker/Dtos/AddNodeStatusDto.cs

@@ -1,8 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.ComponentModel.DataAnnotations;
-using System.Linq;
-using System.Threading.Tasks;
+using System.ComponentModel.DataAnnotations;
 
 namespace DotnetSpider.Broker.Dtos
 {

src/DotnetSpider.Broker/Dtos/BlockDto.cs

@@ -1,9 +1,5 @@
-using DotnetSpider.Common;
-using DotnetSpider.Downloader;
-using System;
+using DotnetSpider.Downloader;
 using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Dtos
 {

src/DotnetSpider.Broker/HttpGlobalExceptionFilter.cs

@@ -1,10 +1,6 @@
 using Microsoft.AspNetCore.Mvc.Filters;
 using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Net;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker
 {

src/DotnetSpider.Broker/Hubs/NodeHub.cs

@@ -2,10 +2,7 @@
 using DotnetSpider.Broker.Dtos;
 using DotnetSpider.Broker.Services;
 using Microsoft.AspNetCore.SignalR;
-using Microsoft.EntityFrameworkCore;
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Hubs

src/DotnetSpider.Broker/Hubs/WorkerHub.cs

@@ -1,10 +1,6 @@
-using DotnetSpider.Broker.Data;
-using DotnetSpider.Broker.Services;
+using DotnetSpider.Broker.Services;
 using Microsoft.AspNetCore.SignalR;
-using Microsoft.EntityFrameworkCore;
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Hubs

src/DotnetSpider.Broker/Models/ErrorViewModel.cs

@@ -1,5 +1,3 @@
-using System;
-
 namespace DotnetSpider.Broker.Models
 {
 	public class ErrorViewModel

src/DotnetSpider.Broker/Program.cs

@@ -1,12 +1,8 @@
 using System;
-using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Threading.Tasks;
 using Microsoft.AspNetCore;
 using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.Configuration;
-using Microsoft.Extensions.Logging;
 using Serilog;
 using Serilog.Events;
 

src/DotnetSpider.Broker/ServiceCollectionExtensions.cs

@@ -1,9 +1,5 @@
 using DotnetSpider.Broker.Services;
 using Microsoft.Extensions.DependencyInjection;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker
 {

src/DotnetSpider.Broker/Services/INodeService.cs

@@ -1,7 +1,4 @@
-using DotnetSpider.Broker.Data;
-using System;
-using System.Collections.Generic;
-using System.Linq;
+using System;
 using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Services

src/DotnetSpider.Broker/Services/INodeStatusService.cs

@@ -1,7 +1,4 @@
 using DotnetSpider.Broker.Data;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Services

src/DotnetSpider.Broker/Services/IWorkerService.cs

@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
+using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Services
 {

src/DotnetSpider.Broker/Services/NodeService.cs

@@ -1,8 +1,6 @@
 using DotnetSpider.Broker.Data;
 using Microsoft.EntityFrameworkCore;
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Services

src/DotnetSpider.Broker/Services/NodeStatusService.cs

@@ -1,7 +1,4 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading.Tasks;
+using System.Threading.Tasks;
 using DotnetSpider.Broker.Data;
 
 namespace DotnetSpider.Broker.Services

src/DotnetSpider.Broker/Services/WorkerService.cs

@@ -1,8 +1,5 @@
 using DotnetSpider.Broker.Data;
 using Microsoft.EntityFrameworkCore;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace DotnetSpider.Broker.Services