Skip to content

Commit 95bd0c4

Browse files
zlzforeverzlzforever
committed
update build & publish scripts
1 parent bf04db9 commit 95bd0c4

12 files changed

+52
-56
lines changed

‎.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ selenium-debug.log
1212
**/wwwroot/lib
1313

1414
*.min.js
15+
*.min.css
1516

1617
yarn.lock
1718

‎DotnetSpider.sln

+12-10
Original file line numberDiff line numberDiff line change
@@ -14,16 +14,18 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "DotnetSpider.HBase", "src\D
1414
EndProject
1515
Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{724DCFC7-FA90-4AA7-8A2E-32FACCDEB38C}"
1616
ProjectSection(SolutionItems) = preProject
17-
.editorconfig = .editorconfig
18-
.gitignore = .gitignore
19-
azure-pipelines.yml = azure-pipelines.yml
20-
Directory.Build.props = Directory.Build.props
21-
LICENSE.txt = LICENSE.txt
22-
package.props = package.props
23-
README.md = README.md
24-
runtests.sh = runtests.sh
25-
publish_agent.sh = publish_agent.sh
26-
publish_portal.sh = publish_portal.sh
17+
.editorconfig = .editorconfig
18+
.gitignore = .gitignore
19+
azure-pipelines.yml = azure-pipelines.yml
20+
Directory.Build.props = Directory.Build.props
21+
LICENSE.txt = LICENSE.txt
22+
package.props = package.props
23+
README.md = README.md
24+
runtests.sh = runtests.sh
25+
publish_agent.sh = publish_agent.sh
26+
publish_portal.sh = publish_portal.sh
27+
publish_spiders.sh = publish_spiders.sh
28+
publish.sh = publish.sh
2729
build_spiders.sh = build_spiders.sh
2830
build_agent.sh = build_agent.sh
2931
build_portal.sh = build_portal.sh

‎README.md

+19-18
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,11 @@
66
[![NuGet](https://img.shields.io/nuget/vpre/DotnetSpider.svg)](https://www.nuget.org/packages/DotnetSpider)
77
[![Member project of .NET Core Community](https://img.shields.io/badge/member%20project%20of-NCC-9e20c9.svg)](https://github.com/dotnetcore)
88
[![GitHub license](https://img.shields.io/github/license/dotnetcore/DotnetSpider.svg)](https://github.com/dotnetcore/DotnetSpider/blob/master/LICENSE.txt)
9+
[![FOSSA Status](https://app.fossa.com/api/projects/git%2Bgithub.com%2Fdotnetcore%2FDotnetSpider.svg?type=shield)](https://app.fossa.com/projects/git%2Bgithub.com%2Fdotnetcore%2FDotnetSpider?ref=badge_shield)
910

1011
DotnetSpider, a .NET Standard web crawling library. It is lightweight, efficient and fast high-level web crawling & scraping framework.
1112

12-
If you want get latest beta packages, you should add the myget feed:
13+
If you want get latest beta packages, you should add the myget feed:
1314

1415
```
1516
<add key="myget.org" value="https://www.myget.org/F/zlzforever/api/v3/index.json" protocolVersion="3" />
@@ -43,21 +44,21 @@ If you want get latest beta packages, you should add the myget feed:
4344
9. MongoDb (option)
4445

4546
docker run --name mongo -d -p 27017:27017 --restart always mongo
46-
47+
4748
10. RabbitMQ
4849

4950
docker run -d --restart always --name rabbimq -p 4369:4369 -p 5671-5672:5671-5672 -p 25672:25672 -p 15671-15672:15671-15672 \
5051
-e RABBITMQ_DEFAULT_USER=user -e RABBITMQ_DEFAULT_PASS=password \
5152
rabbitmq:3-management
52-
53+
5354
11. Docker remote api for mac
5455

5556
docker run -d --restart always --name socat -v /var/run/docker.sock:/var/run/docker.sock -p 2376:2375 bobrik/socat TCP4-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock
5657

5758
12. HBase
5859

59-
docker run -d --restart always --name hbase -p 20550:8080 -p 8085:8085 -p 9090:9090 -p 9095:9095 -p 16010:16010 dajobe/hbase
60-
60+
docker run -d --restart always --name hbase -p 20550:8080 -p 8085:8085 -p 9090:9090 -p 9095:9095 -p 16010:16010 dajobe/hbase
61+
6162
### MORE DOCUMENTS
6263

6364
https://github.com/dotnetcore/DotnetSpider/wiki
@@ -83,12 +84,12 @@ https://github.com/dotnetcore/DotnetSpider/wiki
8384
builder.UseQueueDistinctBfsScheduler<HashSetDuplicateRemover>();
8485
await builder.Build().RunAsync();
8586
}
86-
87+
8788
public EntitySpider(IOptions<SpiderOptions> options, SpiderServices services, ILogger<Spider> logger) : base(
8889
options, services, logger)
8990
{
9091
}
91-
92+
9293
protected override async Task InitializeAsync(CancellationToken stoppingToken)
9394
{
9495
AddDataFlow(new DataParser<CnblogsEntry>());
@@ -97,12 +98,12 @@ https://github.com/dotnetcore/DotnetSpider/wiki
9798
new Request("https://news.cnblogs.com/n/page/1/", new Dictionary<string, string> {{"网站", "博客园"}}),
9899
new Request("https://news.cnblogs.com/n/page/2/", new Dictionary<string, string> {{"网站", "博客园"}}));
99100
}
100-
101+
101102
protected override (string Id, string Name) GetIdAndName()
102103
{
103104
return (Guid.NewGuid().ToString(), "博客园");
104105
}
105-
106+
106107
[Schema("cnblogs", "news")]
107108
[EntitySelector(Expression = ".//div[@class='news_block']", Type = SelectorType.XPath)]
108109
[GlobalValueSelector(Expression = ".//a[@class='current']", Name = "类别", Type = SelectorType.XPath)]
@@ -114,37 +115,37 @@ https://github.com/dotnetcore/DotnetSpider/wiki
114115
HasIndex(x => x.Title);
115116
HasIndex(x => new {x.WebSite, x.Guid}, true);
116117
}
117-
118+
118119
public int Id { get; set; }
119-
120+
120121
[Required]
121122
[StringLength(200)]
122123
[ValueSelector(Expression = "类别", Type = SelectorType.Environment)]
123124
public string Category { get; set; }
124-
125+
125126
[Required]
126127
[StringLength(200)]
127128
[ValueSelector(Expression = "网站", Type = SelectorType.Environment)]
128129
public string WebSite { get; set; }
129-
130+
130131
[StringLength(200)]
131132
[ValueSelector(Expression = "//title")]
132133
[ReplaceFormatter(NewValue = "", OldValue = " - 博客园")]
133134
public string Title { get; set; }
134-
135+
135136
[StringLength(40)]
136137
[ValueSelector(Expression = "GUID", Type = SelectorType.Environment)]
137138
public string Guid { get; set; }
138-
139+
139140
[ValueSelector(Expression = ".//h2[@class='news_entry']/a")]
140141
public string News { get; set; }
141-
142+
142143
[ValueSelector(Expression = ".//h2[@class='news_entry']/a/@href")]
143144
public string Url { get; set; }
144-
145+
145146
[ValueSelector(Expression = ".//div[@class='entry_summary']")]
146147
public string PlainText { get; set; }
147-
148+
148149
[ValueSelector(Expression = "DATETIME", Type = SelectorType.Environment)]
149150
public DateTime CreationTime { get; set; }
150151
}

‎build_portal.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
cd src/DotnetSpider.Portal & yarn install
2+
dotnet publish -c Release
3+
cp -r src/DotnetSpider.Portal/bin/Release/netcoreapp3.1/publish/ dockerfile/portal/out
4+
cd dockerfile/portal
5+
docker build -t dotnetspider/portal:latest .

‎docker-compose/portal.yml

+1-9
Original file line numberDiff line numberDiff line change
@@ -4,14 +4,6 @@ services:
44
dotnetspider.portal:
55
image: 'dotnetspider/portal:latest'
66
restart: always
7-
hostname: dotnetspider.portal
87
container_name: dotnetspider.portal
98
ports:
10-
- '7896:7896'
11-
environment:
12-
- ConnectionString=Database='dotnetspider';Data Source=my.com;password=1qazZAQ!;User ID=root;Port=3306;
13-
- Docker=http://my.com:2376
14-
- DockerVolumes=/storage/var/lib/dotnetspider/logs:/logs;/etc/hosts:/etc/hosts
15-
- quartz.dataSource.myDs.connectionString=Database='dotnetspider';Data Source=my.com;password=1qazZAQ!;User ID=root;Port=3306;
16-
volumes:
17-
- /storage/var/lib/dotnetspider/portal/logs:/logs
9+
- '7896:7896'

‎dockerfile/portal/Dockerfile

+2-8
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,4 @@
1-
FROM mcr.microsoft.com/dotnet/core/sdk:3.1 AS build-env
1+
FROM mcr.microsoft.com/dotnet/core/aspnet:3.1
22
WORKDIR /portal
3-
COPY . ./
4-
RUN dotnet publish -c Release -o out
5-
6-
FROM mcr.microsoft.com/dotnet/core/runtime:3.1
7-
WORKDIR /portal
8-
COPY --from=build-env /portal/out/ .
9-
3+
COPY ./out .
104
ENTRYPOINT ["dotnet", "DotnetSpider.Portal.dll"]

‎publish.sh

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
sh publish_agent.sh
2+
sh publish_portal.sh
3+
sh publish_spiders.sh

‎publish_agent.sh

+2-4
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,2 @@
1-
cd src/DotnetSpider.Agent
2-
docker build -t dotnetspider/agent:latest .
3-
rm -rf src/DotnetSpider.Agent/out
4-
docker push dotnetspider/downloader-agent:latest
1+
sh build_agent.sh
2+
docker push dotnetspider/agent:latest

‎publish_portal.sh

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,2 @@
1-
cd src/DotnetSpider.Portal
2-
docker build -t dotnetspider/portal:latest .
3-
rm -rf src/DotnetSpider.Portal/out
1+
sh build_portal.sh
42
docker push dotnetspider/portal:latest

‎publish_spiders.sh

+2
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,2 @@
1+
sh build_spiders.sh
2+
docker push dotnetspider/spiders:latest

‎src/DotnetSpider.Portal/DotnetSpider.Portal.csproj

+3-3
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@
1010
<ItemGroup>
1111
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="7.0.0" />
1212
<PackageReference Include="Docker.DotNet" Version="3.125.2" />
13-
<PackageReference Include="BuildBundlerMinifier" Version="3.2.435" />
13+
<PackageReference Include="BuildBundlerMinifier" Version="3.2.447" />
1414
<PackageReference Include="Pomelo.EntityFrameworkCore.MySql" Version="3.1.1" />
1515
<PackageReference Include="Quartz" Version="3.0.7" />
1616
<PackageReference Include="Quartz.AspNetCore.MySqlConnector" Version="1.0.4" />
@@ -23,8 +23,8 @@
2323
<PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="3.1.3" />
2424
<PackageReference Include="System.Text.Json" Version="4.7.1" />
2525
<PackageReference Include="Microsoft.AspNetCore.Mvc.Razor.RuntimeCompilation" Version="3.1.3" />
26-
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.2" />
27-
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.2" />
26+
<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.3" />
27+
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="3.1.3" />
2828
</ItemGroup>
2929

3030

‎src/DotnetSpider.Tests/DotnetSpider.Tests.csproj

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
<PackageReference Include="System.Text.Json" Version="4.7.1" />
1212
<PackageReference Include="xunit" Version="2.4.1" />
1313
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.1" />
14-
<PackageReference Include="coverlet.collector" Version="1.2.0" />
14+
<PackageReference Include="coverlet.collector" Version="1.2.1" />
1515
<PackageReference Include="Moq" Version="4.13.1" />
1616
</ItemGroup>
1717

0 commit comments

Comments
 (0)