diff --git a/.gitignore b/.gitignore index 757fee3..a8d0298 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,5 @@ -/.idea \ No newline at end of file +/.idea +*/*/Gopkg.lock +*/*/_dev +vendor +*/*/go.sum \ No newline at end of file diff --git a/Config/mysql.toml b/Config/mysql.toml deleted file mode 100644 index 3d37262..0000000 --- a/Config/mysql.toml +++ /dev/null @@ -1,2 +0,0 @@ -Source = "user:password@tcp(ip:port)/database?charset=utf8mb4" -Driver = "mysql" \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fadf709 --- /dev/null +++ b/Makefile @@ -0,0 +1,43 @@ +.PHONY: dev release push publish + + +PWD := $(shell pwd) +USER := $(shell id -nu) + +DEV_NAME = $(USER)-$(PWD) + +all: dev + +dev: + cd docker/dev && docker-compose -p "$(DEV_NAME)" down && docker-compose -p "$(DEV_NAME)" up --force-recreate + +dep: + docker run --rm \ + -v $(PWD)/src/app:/go/src/app \ + -v /tmp/golang-dep:/go/pkg/dep \ + -w /go/src/app \ + -ti docker.io/library/golang:1.13.0-buster \ + sh -c 'curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh && dep ensure -v' + +gomod: + docker run --rm \ + -v $(PWD)/src/app:/go/src/app \ + -v /tmp/golang-mod:/go/pkg/mod \ + -w /go/src/app \ + -e GOPROXY=https://goproxy.cn \ + -ti docker.io/library/golang:1.13.0-buster \ + go mod vendor + +build-server: + docker run --rm \ + -v $(PWD)/src/app:/go/src/app \ + -w /go/src/app \ + -ti docker.io/library/golang:1.13.0-buster \ + go build -o ./App/Server ./App/Server.go + +build-gethot: + docker run --rm \ + -v $(PWD)/src/app:/go/src/app \ + -w /go/src/app \ + -ti docker.io/library/golang:1.13.0-buster \ + go build -o ./App/GetHot ./App/GetHot.go \ No newline at end of file diff --git a/README.md b/README.md index bcaee59..2d7f154 100644 --- a/README.md +++ b/README.md @@ -6,59 +6,53 @@ **今日热榜是一个获取各大热门网站热门头条的聚合网站，使用Go语言编写，多协程异步快速抓取信息，预览:[https://www.printf520.com/hot.html][热榜]** ![DeepinScrot-4337.png](https://i.loli.net/2019/08/05/PjX2nqWAgM5xsL4.png) -### 安装教程 +### 开发教程 -1. 编译 - - ``` - cd {root_path} # 项目根目录 - go build -o ./App/GetHot App/GetHot.go - go build -o ./App/Server App/Server.go - ``` - -2. 创建数据库，如 `news`，执行database.sql创建表，更改配置文件`Config/mysql.toml` - -3. 编辑文件 `Html/js/blog/globalConfig.js` - - ``` - const ServerIp = 'http://{your_domain}:9090' // 替换成服务器域名 - ``` - -4. 部署定时任务/App/GetHot.go爬虫程序，且以守护进程的方式执行Server.go - - ``` - crontab -e # 添加一行 0 */1 * * * {root_path}/App/GetHot - nohup {root_path}/App/Server & - ``` - -5. 测试 - - - 打开`http://{yourdomain}:9090/` 即可访问今日热榜 +1. 将 `/your/path/to/TopList/src` 添加到 $GOPATH，（如果用IDE开发，需要代码提示的话） +2. `cd {root_path}` # 项目根目录 +3. `cp docker/dev/default.env docker/dev/.env` # 复制docker-compose环境文件 +4. `make gomod` 或 `make dep` # 安装依赖包, `make gomod`是使用 `gomodule`, `make dep`是使用 `dep` 工具 +5. `make dev` # 启动。会根据database.sql自动创建数据库，同时使用 [AIR](https://github.com/cosmtrek/air), 不需要重启即可热重载 +6. 等待服务启动完毕，打开`http://{yourdomain}:{DEV_PORT}/` 即可访问今日热榜 +### 编译教程 +``` +cd {root_path} # 项目根目录 +make build-server +make build-gethot +``` ### 目录说明 ``` TopList/ -├── App -│   ├── GetHot.go 爬虫程序需要Cron定时任务执行 -│   └── Server.go Server程序需要守护进程的方式执行 -├── Common -│   ├── Db.go DB组件 -│   └── Message.go -├── Config -│   ├── MySql.go mysql配置读取组件 -│   └── mysql.toml mysql配置文件需要手动配置 -├── Cron -│   ├── GetHot.sh 爬虫Cron程序可以是每小时执行一次 -│   └── README.md -├── database.sql 数据库建表文件 -├── Html -│   ├── css -│   ├── hot.html 前端热榜展示网页 -│   └── js -│   -└── README.md +├── docker +│   └── dev +│ ├── air # air 的可执行文件 +│ ├── air-gethot.conf # GetHot 的 air 热加载配置 +│ ├── air-server.conf # Server 的 air 热加载配置 +│ ├── default.env # docker-compose 默认的环境变量 +│ └── docker-compose.yaml # docker-compose 配置文件 +└── src + └── app + ├── App + │   ├── GetHot.go 爬虫程序需要Cron定时任务执行 + │   └── Server.go Server程序需要守护进程的方式执行 + ├── Common + │   ├── Db.go DB组件 + │   ├── Message.go + │  └── database.sql 数据库建表文件 + ├── Config + │   ├── MySql.go mysql配置读取组件 + │   └── mysql.toml mysql配置文件需要手动配置 + ├── Cron + │   ├── GetHot.sh 爬虫Cron程序可以是每小时执行一次 + │   └── README.md + ├── Html + │   ├── css + │   ├── hot.html 前端热榜展示网页 + │   └── js + └── README.md ``` ### API说明 diff --git a/docker/dev/.env b/docker/dev/.env new file mode 100644 index 0000000..075bc04 --- /dev/null +++ b/docker/dev/.env @@ -0,0 +1,5 @@ +DEV_USER=h1z3y3 +DEV_PORT=9090 +MYSQL_DATABASE=toplist +MYSQL_ROOT_PASSWORD=a1b2c3d4e +MYSQL_ROOT_HOST=% \ No newline at end of file diff --git a/docker/dev/air b/docker/dev/air new file mode 100755 index 0000000..fb4948f Binary files /dev/null and b/docker/dev/air differ diff --git a/docker/dev/air-gethot.conf b/docker/dev/air-gethot.conf new file mode 100644 index 0000000..6ee48e2 --- /dev/null +++ b/docker/dev/air-gethot.conf @@ -0,0 +1,24 @@ +# conf for [Air](https://github.com/cosmtrek/air) in TOML format + +# working directory +root = "." # . or absolute path, please note that the directories following must be under root +watch_dir = "" # optional! if no watch_dir, use root +tmp_dir = "_dev/" + +[build] +bin = "./_dev/run-build" #这里写执行时需要的参数 +cmd = "go build -o ./_dev/run-build ./App/GetHot.go" +log = "server-errors.log" # this log file places in your tmp_dir +include_ext = ["go", "tpl", "tmpl", "json", "toml"] +exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"] +delay = 800 # ms + +[log] +time = true + +[color] +main = "magenta" +watcher = "cyan" +build = "yellow" +runner = "green" +app = "white" diff --git a/docker/dev/air-server.conf b/docker/dev/air-server.conf new file mode 100644 index 0000000..b629d33 --- /dev/null +++ b/docker/dev/air-server.conf @@ -0,0 +1,24 @@ +# conf for [Air](https://github.com/cosmtrek/air) in TOML format + +# working directory +root = "." # . or absolute path, please note that the directories following must be under root +watch_dir = "" # optional! if no watch_dir, use root +tmp_dir = "_dev/" + +[build] +bin = "./_dev/run-build" #这里写执行时需要的参数 +cmd = "go build -o ./_dev/run-build ./App/Server.go" +log = "server-errors.log" # this log file places in your tmp_dir +include_ext = ["go", "tpl", "tmpl", "json", "toml"] +exclude_dir = ["assets", "tmp", "vendor", "frontend/node_modules"] +delay = 800 # ms + +[log] +time = true + +[color] +main = "magenta" +watcher = "cyan" +build = "yellow" +runner = "green" +app = "white" diff --git a/docker/dev/default.env b/docker/dev/default.env new file mode 100644 index 0000000..075bc04 --- /dev/null +++ b/docker/dev/default.env @@ -0,0 +1,5 @@ +DEV_USER=h1z3y3 +DEV_PORT=9090 +MYSQL_DATABASE=toplist +MYSQL_ROOT_PASSWORD=a1b2c3d4e +MYSQL_ROOT_HOST=% \ No newline at end of file diff --git a/docker/dev/docker-compose.yaml b/docker/dev/docker-compose.yaml new file mode 100644 index 0000000..d37febf --- /dev/null +++ b/docker/dev/docker-compose.yaml @@ -0,0 +1,66 @@ +version: '2.0' +services: + server: + image: docker.io/library/golang:1.13.0-buster + container_name: toplist-server-${DEV_PORT} + working_dir: /go/src/app + command: + - sh + - -c + - | + #使用air做动态代码检查，代码变更后自动重新build并运行 + #!!!禁止用于生产环境 + #curl -fLo /usr/local/bin/air https://raw.githubusercontent.com/cosmtrek/air/master/bin/linux/air + echo "Asia/Shanghai" >> /etc/timezone + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + chmod +x /usr/local/bin/air + cd /go/src/app + air -c /etc/air/air.conf + volumes: + - ../../src:/go/src/ + #用于开发环境动态加载代码变更 + - ./air-server.conf:/etc/air/air.conf + - ./air:/usr/local/bin/air + links: + - mysql:mysql + ports: + - "${DEV_PORT}:9090" + gethot: + image: docker.io/library/golang:1.13.0-buster + container_name: toplist-gethot-crontab + working_dir: /go/src/app + command: + - sh + - -c + - | + #使用air做动态代码检查，代码变更后自动重新build并运行 + #!!!禁止用于生产环境 + #curl -fLo /usr/local/bin/air https://raw.githubusercontent.com/cosmtrek/air/master/bin/linux/air + echo "Asia/Shanghai" >> /etc/timezone + cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime + chmod +x /usr/local/bin/air + cd /go/src/app + air -c /etc/air/air.conf + volumes: + - ../../src:/go/src/ + #用于开发环境动态加载代码变更 + - ./air-gethot.conf:/etc/air/air.conf + - ./air:/usr/local/bin/air + links: + - mysql:mysql + mysql: + image: docker.io/library/mysql:8 + container_name: toplist-mysql-3306 + environment: + MYSQL_DATABASE: "${MYSQL_DATABASE}" + MYSQL_ROOT_PASSWORD: "${MYSQL_ROOT_PASSWORD}" + MYSQL_ROOT_HOST: "${MYSQL_ROOT_HOST}" + volumes: + - ./_dev/mysql:/var/lib/mysql:z + - ../../src/app/Common/database.sql:/docker-entrypoint-initdb.d/database.sql + expose: + - "3306" +networks: + default: + external: + name: ${DEV_USER} diff --git a/App/GetHot.go b/src/app/App/GetHot.go similarity index 99% rename from App/GetHot.go rename to src/app/App/GetHot.go index 2239e60..42dc00d 100644 --- a/App/GetHot.go +++ b/src/app/App/GetHot.go @@ -1,7 +1,7 @@ package main import ( - "../Common" + "app/Common" "bytes" "encoding/json" "fmt" diff --git a/App/Server.go b/src/app/App/Server.go similarity index 92% rename from App/Server.go rename to src/app/App/Server.go index 0466198..8bbf4a5 100644 --- a/App/Server.go +++ b/src/app/App/Server.go @@ -7,8 +7,8 @@ import ( "regexp" "text/template" - "../Common" - "../Config" + "app/Common" + "app/Config" ) func GetTypeInfo(w http.ResponseWriter, r *http.Request) { @@ -61,20 +61,23 @@ func main() { http.HandleFunc("/GetConfig", GetConfig) // 设置访问的路由 // 静态资源 - http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("../Html/css/")))) - http.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir("../Html/js/")))) + http.Handle("/css/", http.StripPrefix("/css/", http.FileServer(http.Dir("Html/css/")))) + http.Handle("/js/", http.StripPrefix("/js/", http.FileServer(http.Dir("Html/js/")))) // 首页 http.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) { - t, err := template.ParseFiles("../Html/hot.html") + t, err := template.ParseFiles("Html/hot.html") if err != nil { log.Println("err") } t.Execute(res, nil) }) + fmt.Println("Server Start At :9090") + err := http.ListenAndServe(":9090", nil) // 设置监听的端口 if err != nil { log.Fatal("ListenAndServe: ", err) } + } diff --git a/Common/Db.go b/src/app/Common/Db.go similarity index 99% rename from Common/Db.go rename to src/app/Common/Db.go index fc14948..75dd0b2 100644 --- a/Common/Db.go +++ b/src/app/Common/Db.go @@ -1,7 +1,7 @@ package Common import ( - "../Config" + "app/Config" "database/sql" "fmt" _ "github.com/go-sql-driver/mysql" diff --git a/Common/Message.go b/src/app/Common/Message.go similarity index 100% rename from Common/Message.go rename to src/app/Common/Message.go diff --git a/Common/database.sql b/src/app/Common/database.sql similarity index 99% rename from Common/database.sql rename to src/app/Common/database.sql index a620813..6ce3522 100644 --- a/Common/database.sql +++ b/src/app/Common/database.sql @@ -19,6 +19,10 @@ -- Table structure for table `hotData2` -- +CREATE DATABASE IF NOT EXISTS `toplist` CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci; + +USE `toplist`; + DROP TABLE IF EXISTS `hotData2`; /*!40101 SET @saved_cs_client = @@character_set_client */; /*!40101 SET character_set_client = utf8 */; diff --git a/Config/MySql.go b/src/app/Config/MySql.go similarity index 100% rename from Config/MySql.go rename to src/app/Config/MySql.go diff --git a/src/app/Config/mysql.toml b/src/app/Config/mysql.toml new file mode 100644 index 0000000..855b73b --- /dev/null +++ b/src/app/Config/mysql.toml @@ -0,0 +1,2 @@ +Source = "root:a1b2c3d4e@tcp(mysql:3306)/toplist?charset=utf8mb4" +Driver = "mysql" \ No newline at end of file diff --git a/Cron/GetHot.sh b/src/app/Cron/GetHot.sh similarity index 100% rename from Cron/GetHot.sh rename to src/app/Cron/GetHot.sh diff --git a/Cron/README.md b/src/app/Cron/README.md similarity index 100% rename from Cron/README.md rename to src/app/Cron/README.md diff --git a/src/app/Gopkg.toml b/src/app/Gopkg.toml new file mode 100644 index 0000000..488c252 --- /dev/null +++ b/src/app/Gopkg.toml @@ -0,0 +1,26 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + diff --git a/Html/css/bootstrap.min.css b/src/app/Html/css/bootstrap.min.css similarity index 100% rename from Html/css/bootstrap.min.css rename to src/app/Html/css/bootstrap.min.css diff --git a/Html/css/custom.css b/src/app/Html/css/custom.css similarity index 100% rename from Html/css/custom.css rename to src/app/Html/css/custom.css diff --git a/Html/css/ext.css b/src/app/Html/css/ext.css similarity index 100% rename from Html/css/ext.css rename to src/app/Html/css/ext.css diff --git a/Html/css/fontawesome-all.min.css b/src/app/Html/css/fontawesome-all.min.css similarity index 100% rename from Html/css/fontawesome-all.min.css rename to src/app/Html/css/fontawesome-all.min.css diff --git a/Html/css/hot.css b/src/app/Html/css/hot.css similarity index 100% rename from Html/css/hot.css rename to src/app/Html/css/hot.css diff --git a/Html/css/ionicons.min.css b/src/app/Html/css/ionicons.min.css similarity index 100% rename from Html/css/ionicons.min.css rename to src/app/Html/css/ionicons.min.css diff --git a/Html/css/load.css b/src/app/Html/css/load.css similarity index 100% rename from Html/css/load.css rename to src/app/Html/css/load.css diff --git a/Html/css/pace.css b/src/app/Html/css/pace.css similarity index 100% rename from Html/css/pace.css rename to src/app/Html/css/pace.css diff --git a/Html/css/style.css b/src/app/Html/css/style.css similarity index 100% rename from Html/css/style.css rename to src/app/Html/css/style.css diff --git a/Html/css/uikit.min.css b/src/app/Html/css/uikit.min.css similarity index 100% rename from Html/css/uikit.min.css rename to src/app/Html/css/uikit.min.css diff --git a/Html/hot.html b/src/app/Html/hot.html similarity index 100% rename from Html/hot.html rename to src/app/Html/hot.html diff --git a/Html/js/2D/L2Dwidget.0.min.js b/src/app/Html/js/2D/L2Dwidget.0.min.js similarity index 100% rename from Html/js/2D/L2Dwidget.0.min.js rename to src/app/Html/js/2D/L2Dwidget.0.min.js diff --git a/Html/js/2D/L2Dwidget.1.min.js.js b/src/app/Html/js/2D/L2Dwidget.1.min.js.js similarity index 100% rename from Html/js/2D/L2Dwidget.1.min.js.js rename to src/app/Html/js/2D/L2Dwidget.1.min.js.js diff --git a/Html/js/blog/20181113171647745.gif b/src/app/Html/js/blog/20181113171647745.gif similarity index 100% rename from Html/js/blog/20181113171647745.gif rename to src/app/Html/js/blog/20181113171647745.gif diff --git a/Html/js/blog/globalConfig.js b/src/app/Html/js/blog/globalConfig.js similarity index 100% rename from Html/js/blog/globalConfig.js rename to src/app/Html/js/blog/globalConfig.js diff --git a/Html/js/blog/hot.js b/src/app/Html/js/blog/hot.js similarity index 100% rename from Html/js/blog/hot.js rename to src/app/Html/js/blog/hot.js diff --git a/Html/js/blog/lazyload.gif b/src/app/Html/js/blog/lazyload.gif similarity index 100% rename from Html/js/blog/lazyload.gif rename to src/app/Html/js/blog/lazyload.gif diff --git a/Html/js/blog/mine.js b/src/app/Html/js/blog/mine.js similarity index 100% rename from Html/js/blog/mine.js rename to src/app/Html/js/blog/mine.js diff --git a/Html/js/blog/shake.js b/src/app/Html/js/blog/shake.js similarity index 100% rename from Html/js/blog/shake.js rename to src/app/Html/js/blog/shake.js diff --git a/Html/js/blog/timg.gif b/src/app/Html/js/blog/timg.gif similarity index 100% rename from Html/js/blog/timg.gif rename to src/app/Html/js/blog/timg.gif diff --git a/Html/js/bootstrap.min.js b/src/app/Html/js/bootstrap.min.js similarity index 100% rename from Html/js/bootstrap.min.js rename to src/app/Html/js/bootstrap.min.js diff --git a/Html/js/jquery-2.1.3.min.js b/src/app/Html/js/jquery-2.1.3.min.js similarity index 100% rename from Html/js/jquery-2.1.3.min.js rename to src/app/Html/js/jquery-2.1.3.min.js diff --git a/Html/js/modernizr.custom.js b/src/app/Html/js/modernizr.custom.js similarity index 100% rename from Html/js/modernizr.custom.js rename to src/app/Html/js/modernizr.custom.js diff --git a/Html/js/pace.min.js b/src/app/Html/js/pace.min.js similarity index 100% rename from Html/js/pace.min.js rename to src/app/Html/js/pace.min.js diff --git a/Html/js/script.js b/src/app/Html/js/script.js similarity index 100% rename from Html/js/script.js rename to src/app/Html/js/script.js diff --git a/Html/js/uikit.min.js b/src/app/Html/js/uikit.min.js similarity index 100% rename from Html/js/uikit.min.js rename to src/app/Html/js/uikit.min.js diff --git a/src/app/go.mod b/src/app/go.mod new file mode 100644 index 0000000..be26eaf --- /dev/null +++ b/src/app/go.mod @@ -0,0 +1,12 @@ +module app + +go 1.13 + +require ( + github.com/BurntSushi/toml v0.3.1 + github.com/PuerkitoBio/goquery v1.5.0 + github.com/bitly/go-simplejson v0.5.0 + github.com/go-sql-driver/mysql v1.4.1 + golang.org/x/text v0.3.2 + google.golang.org/appengine v1.6.3 // indirect +)