Skip to content

Commit 2a26a2e

Browse files
author
邹嵩
committed
release 3.0.0
1 parent 1722c30 commit 2a26a2e

File tree

122 files changed

+932
-1255
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

122 files changed

+932
-1255
lines changed

‎README.md

+4-8
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,10 @@ DotnetSpider, a .NET Standard web crawling library similar to WebMagic and Scrap
2121

2222
### OPTIONAL ENVIROMENT
2323

24-
- Run distributed crawler. [Download Redis for windows](https://github.com/MSOpenTech/redis/releases)
24+
- Distributed crawler. [Download Redis for windows](https://github.com/MSOpenTech/redis/releases)
2525
- SqlServer.
2626
- PostgreSQL.
2727
- MongoDb
28-
- Cassandra
2928

3029
### MORE DOCUMENTS
3130

@@ -53,7 +52,7 @@ https://github.com/dotnetcore/DotnetSpider/wiki
5352

5453
private class Spider : EntitySpider
5554
{
56-
protected override void MyInit(params string[] arguments)
55+
protected override void OnInit(params string[] arguments)
5756
{
5857
var word = "可乐|雪碧";
5958
AddStartUrl(string.Format("http://news.baidu.com/ns?word={0}&tn=news&from=news&cl=2&pn=0&rn=20&ct=1", word), new Dictionary<string, dynamic> { { "Keyword", word } });
@@ -130,15 +129,12 @@ NOTE:
130129

131130
### Storage log and status to database
132131

133-
1. Set SystemConnection in app.config
134-
2. Update nlog.config like https://github.com/dotnetcore/DotnetSpider/blob/master/src/DotnetSpider.Extension.Test/nlog.config
135132

136-
137-
### Web Manager
133+
### DotnetSpider.Hub
138134

139135
https://github.com/zlzforever/DotnetSpider.Hub
140136

141-
1. Dependences a ci platform forexample i used gitlab-ci right now.
137+
1. Dependences a ci platform forexample i used teamcity right now.
142138
2. Dependences Sceduler.NET https://github.com/zlzforever/Scheduler.NET
143139
3. More documents continue...
144140

‎nuget/DotnetSpider.Core.nuspec

-55
This file was deleted.

‎nuget/DotnetSpider.Extension.nuspec

-56
This file was deleted.

‎src/DotnetSpider.Common.Test/RequestTest.cs

+2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
using Newtonsoft.Json;
22
using System.Collections.Generic;
3+
using System.Net.Http;
34
using Xunit;
45

56
namespace DotnetSpider.Common.Test
@@ -24,6 +25,7 @@ public void Request()
2425
Assert.Single(request.Properties);
2526
Assert.Equal(request.Properties["Test"], "Forever");
2627
}
28+
2729
[Fact(DisplayName = "Request_PutExtra")]
2830
public void PutExtra()
2931
{

‎src/DotnetSpider.Common/DotnetSpider.Common.csproj

+3-1
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
<TargetFrameworks>net40;net45;netstandard2.0</TargetFrameworks>
44
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
55
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
6-
<Version>3.0.0-beta1</Version>
6+
<Version>3.0.0</Version>
77
<Authors>[email protected];</Authors>
88
<AssemblyName>DotnetSpider.Common</AssemblyName>
99
<Copyright>Copyright 2018 Lewis Zou</Copyright>
@@ -18,9 +18,11 @@
1818
</PropertyGroup>
1919
<ItemGroup Condition=" '$(TargetFramework)' == 'net40' ">
2020
<Reference Include="Microsoft.CSharp" />
21+
<PackageReference Include=" Microsoft.Net.Http" Version="2.2.29" />
2122
</ItemGroup>
2223
<ItemGroup Condition=" '$(TargetFramework)' == 'net45' ">
2324
<Reference Include="Microsoft.CSharp" />
25+
<PackageReference Include="System.Net.Http" Version="4.3.3" />
2426
</ItemGroup>
2527
<ItemGroup Condition=" '$(TargetFramework)' == 'netstandard2.0' ">
2628
<PackageReference Include="Microsoft.CSharp" Version="4.5.0" />

‎src/DotnetSpider.Common/HttpMethod.cs

-18
This file was deleted.

‎src/DotnetSpider.Common/Request.cs

+3-1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
using System;
22
using System.Collections.Generic;
3+
using System.Net.Http;
34
using Newtonsoft.Json;
5+
using Newtonsoft.Json.Converters;
46

57
namespace DotnetSpider.Common
68
{
@@ -32,7 +34,7 @@ public class Request : IDisposable
3234
/// <summary>
3335
/// 请求链接的方法
3436
/// </summary>
35-
public HttpMethod Method { get; set; }
37+
public HttpMethod Method { get; set; } = HttpMethod.Get;
3638

3739
/// <summary>
3840
/// 链接的优先级, 仅用于优先级队列

‎src/DotnetSpider.Common/TaskName.cs

+1-2
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ namespace DotnetSpider.Common
55
/// <summary>
66
/// 任务名称
77
/// </summary>
8-
[AttributeUsage(AttributeTargets.Class, AllowMultiple = false, Inherited = false)]
8+
[AttributeUsage(AttributeTargets.Class, Inherited = false)]
99
public class TaskName : Attribute
1010
{
1111
/// <summary>
@@ -14,7 +14,6 @@ public class TaskName : Attribute
1414
public string Name
1515
{
1616
get;
17-
private set;
1817
}
1918

2019
/// <summary>

‎src/DotnetSpider.Core.Test/PageTest.cs

+1
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using DotnetSpider.Common;
22
using System.Collections.Generic;
33
using System.Linq;
4+
using System.Net.Http;
45
using Xunit;
56

67
namespace DotnetSpider.Core.Test

‎src/DotnetSpider.Core.Test/SpiderTest.cs

+2-2
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,7 @@ public void WhenNoStartUrl()
153153

154154
internal class TestPipeline : BasePipeline
155155
{
156-
public override void Process(IEnumerable<ResultItems> resultItems, ILogger logger, dynamic sender = null)
156+
public override void Process(IList<ResultItems> resultItems, ILogger logger, dynamic sender = null)
157157
{
158158
foreach (var resultItem in resultItems)
159159
{
@@ -276,7 +276,7 @@ protected override Response DowloadContent(Request request)
276276

277277
internal class FastExitPipeline : BasePipeline
278278
{
279-
public override void Process(IEnumerable<ResultItems> resultItems, ILogger logger, dynamic sender = null)
279+
public override void Process(IList<ResultItems> resultItems, ILogger logger, dynamic sender = null)
280280
{
281281
File.AppendAllLines("FastExit_Result.txt", new[] { resultItems.First().Request.Url.ToString() });
282282
}

‎src/DotnetSpider.Core/DotnetSpider.Core.csproj

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,9 @@
11
<Project Sdk="Microsoft.NET.Sdk">
2-
32
<PropertyGroup>
43
<TargetFrameworks>net40;net45;netstandard2.0</TargetFrameworks>
54
<GeneratePackageOnBuild>true</GeneratePackageOnBuild>
65
<PackageRequireLicenseAcceptance>true</PackageRequireLicenseAcceptance>
7-
<Version>3.0.0-beta1</Version>
6+
<Version>3.0.0</Version>
87
<Authors>[email protected];</Authors>
98
<AssemblyName>DotnetSpider.Core</AssemblyName>
109
<Copyright>Copyright 2018 Lewis Zou</Copyright>

‎src/DotnetSpider.Core/Env.cs

-5
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,6 @@ namespace DotnetSpider.Core
1616
/// </summary>
1717
public static class Env
1818
{
19-
/// <summary>
20-
/// TODO: 原则上此版本号与Nuget包版本号同步, 但是不知道有什么好的自动化更新方法
21-
/// </summary>
22-
public const string Version = "2.0.21";
23-
2419
/// <summary>
2520
/// 从配置文件中读取默认Redis连接字符串的关键字
2621
/// </summary>

‎src/DotnetSpider.Core/ISpider.cs

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ namespace DotnetSpider.Core
99
/// <summary>
1010
/// 爬虫接口定义
1111
/// </summary>
12-
public interface ISpider : IDisposable, IControllable, IAppBase
12+
public interface ISpider : IDisposable, IAppBase
1313
{
1414
/// <summary>
1515
/// 采集站点的信息配置

‎src/DotnetSpider.Core/Infrastructure/Description.cs

+5-5
Original file line numberDiff line numberDiff line change
@@ -11,26 +11,26 @@ public class Description : Attribute
1111
/// <summary>
1212
/// 任务所有者
1313
/// </summary>
14-
public string Owner;
14+
public string Owner { get; set; }
1515

1616
/// <summary>
1717
/// 程序的开发者
1818
/// </summary>
19-
public string Developer;
19+
public string Developer{ get; set; }
2020

2121
/// <summary>
2222
/// 程序的开发时间
2323
/// </summary>
24-
public string Date;
24+
public string Date{ get; set; }
2525

2626
/// <summary>
2727
/// 任务主题
2828
/// </summary>
29-
public string Subject;
29+
public string Subject{ get; set; }
3030

3131
/// <summary>
3232
/// 联系邮箱
3333
/// </summary>
34-
public string Email;
34+
public string Email{ get; set; }
3535
}
3636
}

0 commit comments

Comments
 (0)