修复错误 & 调整编译脚本

Author: zlzforever

build_agent.sh

@@ -1,4 +1,4 @@
 dotnet publish  -c Release
 cp -r src/DotnetSpider.Agent/bin/Release/netcoreapp3.1/publish/ dockerfile/agent/out
-cd dockerfile/agent
+cd dockerfile/agent || exit
 docker build -t dotnetspider/agent:latest .

build_portal.sh

@@ -1,5 +1,6 @@
-cd src/DotnetSpider.Portal & yarn install
+cd src/DotnetSpider.Portal && yarn install
 dotnet publish  -c Release
+cd ../.. || exit
 cp -r src/DotnetSpider.Portal/bin/Release/netcoreapp3.1/publish/ dockerfile/portal/out
-cd dockerfile/portal
+cd dockerfile/portal || exit
 docker build -t dotnetspider/portal:latest .

build_spiders.sh

@@ -1,4 +1,4 @@
 dotnet publish  -c Release
 cp -r src/DotnetSpider.Spiders/bin/Release/netcoreapp3.1/publish/ dockerfile/spiders/out
-cd dockerfile/spiders
+cd dockerfile/spiders || exit
 docker build -t dotnetspider/spiders:latest .

package.props

@@ -4,7 +4,7 @@
     <GeneratePackageOnBuild>true</GeneratePackageOnBuild>
     <PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
     <PackageLicenseFile>LICENSE.txt</PackageLicenseFile>
-    <Version>5.0.0-beta2</Version>
+    <Version>5.0.0-beta3</Version>
     <FileVersion>5.0.0.1</FileVersion>
     <Authors>[email protected];</Authors>
     <Copyright>Copyright 2018 Lewis Zou</Copyright>

src/DotnetSpider.Portal/Views/Agent/Index.cshtml

@@ -27,7 +27,7 @@
             </thead>
             <tbody>
             <tr v-if="items.length===0">
-                <td colspan="7">None</td>
+                <td colspan="8">None</td>
             </tr>
             <tr :id="item.id" v-else v-for="item in items">
                 <td v-if="item.online" class="badge-success"></td>

src/DotnetSpider.Portal/Views/Spider/Index.cshtml

@@ -45,8 +45,8 @@
                 <th>CRON</th>
                 <th>TYPE</th>
                 <th>ENVIONMENT</th>
-                <th>MODIFICATION TIME</th>
-                <th style="width: 275px">ACTION</th>
+                <th>EDIT TIME</th>
+                <th style="width: 360px">ACTION</th>
             </tr>
             </thead>
             <tbody>

src/DotnetSpider.Portal/appsettings.json

@@ -1,13 +1,13 @@
 {
-  "ConnectionString": "Database='dotnetspider2';Data Source=192.168.124.200;password=1qazZAQ!;User ID=root;Port=3306;",
+  "ConnectionString": "Database='dotnetspider2';Data Source=localhost;password=1qazZAQ!;User ID=root;Port=3306;",
   "DatabaseType": "MySql",
   "RabbitMQ": {
     "Exchange": "DOTNET_SPIDER",
-    "Host": "192.168.124.200",
+    "Host": "localhost",
     "UserName": "user",
     "Password": "password"
   },
   "Database": "dotnetspider2",
-  "Docker": "http://192.168.124.200:2376",
+  "Docker": "http://192.168.1.105:2376",
   "DockerVolumes": ""
 }

src/DotnetSpider/Spider.cs

@@ -29,11 +29,12 @@ public abstract partial class Spider :
 	{
 		private readonly List<IDataFlow> _dataFlows;
 		private readonly List<IRequestSupplier> _requestSuppliers;
-		private readonly SpiderOptions _options;
 		private readonly RequestedQueue _requestedQueue;
 		private AsyncMessageConsumer<byte[]> _consumer;
 		private readonly SpiderServices _services;
 
+		protected SpiderOptions Options { get; private set; }
+
 		/// <summary>
 		/// 爬虫标识
 		/// </summary>
@@ -53,8 +54,8 @@ ILogger<Spider> logger
 		{
 			Logger = logger;
 			_services = services;
-			_options = options.Value;
-			_requestedQueue = new RequestedQueue(_options);
+			Options = options.Value;
+			_requestedQueue = new RequestedQueue(Options);
 			_requestSuppliers = new List<IRequestSupplier>();
 			_dataFlows = new List<IDataFlow>();
 		}
@@ -80,15 +81,15 @@ protected virtual (string Id, string Name) GetIdAndName()
 
 		protected IDataFlow GetDefaultStorage()
 		{
-			if (string.IsNullOrWhiteSpace(_options.Storage))
+			if (string.IsNullOrWhiteSpace(Options.Storage))
 			{
 				throw new ArgumentNullException($"Storage is not configured");
 			}
 
-			var type = Type.GetType(_options.Storage);
+			var type = Type.GetType(Options.Storage);
 			if (type == null)
 			{
-				throw new SpiderException($"Type of storage {_options.Storage} not found");
+				throw new SpiderException($"Type of storage {Options.Storage} not found");
 			}
 
 			if (!typeof(StorageBase).IsAssignableFrom(type) && !typeof(EntityStorageBase).IsAssignableFrom(type))
@@ -103,7 +104,7 @@ protected IDataFlow GetDefaultStorage()
 				throw new SpiderException($"Storage {type} didn't implement method CreateFromOptions");
 			}
 
-			var storage = method.Invoke(null, new object[] {_options});
+			var storage = method.Invoke(null, new object[] {Options});
 			if (storage == null)
 			{
 				throw new SpiderException("Create default storage failed");
@@ -186,15 +187,15 @@ protected async Task<int> AddRequestsAsync(IEnumerable<Request> requests)
 
 				// 1. 请求次数超过限制则跳过，并添加失败记录
 				// 2. 默认构造的请求次数为 0， 并且不可用户更改，因此可以保证数据安全性
-				if (request.RequestedTimes > _options.RetriedTimes)
+				if (request.RequestedTimes > Options.RetriedTimes)
 				{
 					await _services.StatisticsClient.IncreaseFailureAsync(Id);
 					continue;
 				}
 
 				// 1. 默认构造的深度为 0， 并且用户不可更改，可以保证数据安全
 				// 2. 当深度超过限制则跳过
-				if (_options.Depth > 0 && request.Depth > _options.Depth)
+				if (Options.Depth > 0 && request.Depth > Options.Depth)
 				{
 					continue;
 				}
@@ -295,7 +296,7 @@ private async Task HandleResponseAsync(Request request, Response response, byte[
 			try
 			{
 				using var scope = _services.ServiceProvider.CreateScope();
-				var context = new DataContext(scope.ServiceProvider, _options, request, response);
+				var context = new DataContext(scope.ServiceProvider, Options, request, response);
 				context.AddData(Consts.ResponseBytes, responseBytes);
 
 				foreach (var dataFlow in _dataFlows)
@@ -318,8 +319,8 @@ private async Task HandleResponseAsync(Request request, Response response, byte[
 
 		private async Task RunAsync(CancellationToken stoppingToken)
 		{
-			var tuple = ComputeIntervalAndDequeueBatch(_options.Speed);
-			var sleepTimeLimit = _options.EmptySleepTime * 1000;
+			var tuple = ComputeIntervalAndDequeueBatch(Options.Speed);
+			var sleepTimeLimit = Options.EmptySleepTime * 1000;
 
 			await Task.Factory.StartNew(async () =>
 			{
@@ -337,7 +338,7 @@ await Task.Factory.StartNew(async () =>
 							await _services.StatisticsClient.PrintAsync(Id);
 						}
 
-						if (_requestedQueue.Count > _options.RequestedQueueCount)
+						if (_requestedQueue.Count > Options.RequestedQueueCount)
 						{
 							if (pausedTime > sleepTimeLimit)
 							{
@@ -434,7 +435,7 @@ private async Task<bool> PublishRequestMessagesAsync(params Request[] requests)
 			{
 				foreach (var request in requests)
 				{
-					if (_options.UseProxy)
+					if (Options.UseProxy)
 					{
 						var proxy = await _services.ProxyPool.GetAsync(70);
 						if (proxy == null)