Add task package (#1)

* Add task package.

* Update task package.

* Make task package works as expected.

* Add task examples.

* Add task stats and error handler.

* Add Coordinator.

* Update coordinator.

* Bugs fix.

* Add more examples.

* Some fixes.

* Add test cases.

* Add go.sum.

* Change coordination implementation to setnx.

* Fix no at time periodic schedule.

* Limit job execution time; limit worker count.

* Add jobLock, add Remove and Clear job methods in scheduler.

* Add ClearJobs; Update OnceJob scheduler implementation.

* Add redis cluster support for coordinator.

* Move Coordinate to Run; add unit tests for Coordinate.

* Extract commonJob from OnceJob and PeriodicJob.

* unexport cron expression type.

* Add JobCount.

* Add timezone support in cron.

* Add Stop for Scheduler.

* Add coordinate for oncejob.

* extract coordinate to commonJob.

* Update unit tests.

* Add GetLatestScheduledTime.

* Move unit test into a separate package.

* Add start flag for scheduler.

* make some functions and methods unexported.

* Add examples for task.

* Change IntervalType to cronIntervalType.

* Add comments for exported items.

* Add task tutorial.

* Update README.

* Update README.

* Update go.sum.

Author: zhoufeng1989

README.md

@@ -1 +1,3 @@
-# gorich
+# gorich
+
+[task tutorial](./task/README.md)

go.mod

@@ -0,0 +1,9 @@
+module github.com/byte-power/gorich
+
+go 1.14
+
+require (
+	github.com/go-redis/redis/v8 v8.3.3
+	github.com/panjf2000/ants/v2 v2.4.3
+	github.com/stretchr/testify v1.6.1
+)

go.sum

@@ -0,0 +1,91 @@
+github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
+github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/go-redis/redis/v8 v8.3.3 h1:e0CL9fsFDK92pkIJH2XAeS/NwO2VuIOAoJvI6yktZFk=
+github.com/go-redis/redis/v8 v8.3.3/go.mod h1:jszGxBCez8QA1HWSmQxJO9Y82kNibbUmeYhKWrBejTU=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2 h1:8mVmC9kjFFmA8H4pKMUhcblgifdkOIXPvbhN1T36q1M=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.3 h1:gph6h/qe9GSUw1NhH1gp+qb+h8rXD8Cy60Z32Qw3ELA=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/panjf2000/ants/v2 v2.4.3 h1:wHghL17YKFanB62QjPQ9o+DuM4q7WrQ7zAhoX8+eBXU=
+github.com/panjf2000/ants/v2 v2.4.3/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+go.opentelemetry.io/otel v0.13.0 h1:2isEnyzjjJZq6r2EKMsFj4TxiQiexsM04AVhwbR/oBA=
+go.opentelemetry.io/otel v0.13.0/go.mod h1:dlSNewoRYikTkotEnxdmuBHgzT+k/idJSfDv/FxEnOY=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0 h1:wBouT66WTYFXdxfVdz9sVWARVd/2vfGcmI45D2gj45M=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.23.0 h1:4MY060fB1DLGMB/7MBTLnwQUY6+F09GEiz6SsrNqyzM=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

task/README.md

@@ -0,0 +1,136 @@
+# task package tutorial
+
+task package implements a job scheduler. Jobs in scheduler can be scheduled to run periodically or once at specific time.
+
+## 1. basic usages
+
+You can add periodic jobs or run once jobs to the default scheduler, and then start to schedule.
+
+Examples as below:
+
+``` go
+package main
+
+import (
+    "github.com/byte-power/gorich/task"
+)
+
+func main() {
+    jobName1 := "once_job1_name"
+    // Add jobName1 to the default scheduler, the job will run immediately after scheduler starts.
+    task.Once(jobName1, sum, 10, 20)
+
+    jobName2 := "once_job2_name"
+    // Add jobName2 to the default scheduler, the job will run 5 seconds after scheduler starts.
+    task.Once(jobName2, sum, 100, 200).Delay(5 * time.Second)
+
+    jobName3 := "periodic_job3_name"
+    // Add jobName3 to the default scheduler, the job will run every 2 days at 10:20:30 (in local timezone by default) after scheduler starts.
+     _, err := task.Periodic(jobName3, sum, 20, 30).EveryDays(2).AtHourInDay(10, 20, 30)
+     if err != nil {
+        return
+     }
+
+    // Start the default scheduler.
+    task.StartScheduler()
+}
+
+func sum(a, b int) int {
+    return a + b
+}
+
+```
+
+## 2. monitor job stats
+
+You can also monitor the scheduled jobs via JobStats.
+
+```go
+package main
+
+import (
+    "github.com/byte-power/gorich/task"
+)
+
+func main() {
+    jobName1 := "once_job1_name"
+    // Add jobName1 to the default scheduler, the job will run immediately after scheduler starts.
+    task.Once(jobName1, sum, 10, 20)
+
+    jobName2 := "once_job2_name"
+    // Add jobName2 to the default scheduler, the job will run 5 seconds after scheduler starts.
+    task.Once(jobName2, sum, 100, 200).Delay(5 * time.Second)
+    go monitorScheduler()
+    task.StartScheduler()
+}
+
+func monitorScheduler() {
+    // handle all job stats
+    allJobStats := task.JobStats()
+    for jobName, jobStats := range allJobStats {
+        fmt.Printf("job %s stat:\n", jobName)
+        for _, stat := range jobStats {
+            fmt.Println(stat.ToMap())
+        }
+    }
+}
+
+func sum(a, b int) int {
+    return a + b
+}
+```
+
+## 3. job coordination
+
+When running periodic jobs in multiple servers, you can use Coordinate to coordinate running  and avoid unnecessary running.
+
+Notice that Coordinate use a lock that will unlock automatically 5 seconds later, so if the job running interval is less than 5 seconds, some runnings will not be allowed.
+
+```go
+package main
+
+import (
+    "github.com/byte-power/gorich/task"
+)
+
+func main () {
+    coordinator := task.NewCoordinatorFromRedis("coordinator1", "localhost:6379")
+    // with redis cluster, use:
+    // task.NewCoordinatorFromRedisCluster("coordinator2", []string{"localhost:30000", "localhost:30001"})
+
+    // starts two schedulers
+    scheduler1 := task.NewScheduler(10)
+    scheduler2 := task.NewScheduler(10)
+
+    name := "coordinate_job"
+    job1 := scheduler1.AddPeriodicJob(name, sum, 1, 2).EverySeconds(10).SetCoordinate(coordinator)
+    job2 := scheduler2.AddPeriodicJob(name, sum, 3, 4).EverySeconds(10).SetCoordinate(coordinator)
+
+    // job1 and job2 will coordinate, only one of them will be scheduled once every 10 seconds
+    go scheduler1.Start()
+    go scheduler2.Start()
+
+    jobStats := job1.Stats()
+    fmt.Println("job1 stats:")
+    for _, stat := range jobStats {
+        fmt.Println(stat.ToMap())
+    }
+
+    jobStats = job2.Stats()
+    fmt.Println("job2 stats:")
+    for _, stat := range jobStats {
+        fmt.Println(stat.ToMap())
+    }
+    // stop schedulers after 10 seconds
+    time.Sleep(10 * time.Second)
+
+    scheduler1.Stop(false)
+    scheduler2.Stop(false)
+}
+
+func sum(a, b int) int {
+    return a + b
+}
+```
+
+See more examples [here](./example_test.go).

task/example_test.go

@@ -0,0 +1,119 @@
+package task_test
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/byte-power/gorich/task"
+)
+
+func Example() {
+	// will run immediately after scheduler starts
+	task.Once("once_job1_name", sum, 10, 20)
+
+	// will run 5 seconds after scheduler starts
+	task.Once("once_job2_name", sum, 100, 200).Delay(5 * time.Second)
+
+	// will run every 2 days at 10:20:30 (in local timezone by default) after scheduler starts
+	_, err := task.Periodic("periodic_job3_name", sum, 20, 30).EveryDays(2).AtHourInDay(10, 20, 30)
+	if err != nil {
+		return
+	}
+
+	// will run every Friday at 10:20:30 (in Asia/Shanghai timezone) after scheduler starts
+	tz, _ := time.LoadLocation("Asia/Shanghai")
+	_, err = task.Periodic("periodic_job4_name", sum, 20, 40).EveryFridays(1).SetTimeZone(tz).AtHourInDay(10, 20, 30)
+	if err != nil {
+		return
+	}
+
+	// will run every 2 hours at 20:30
+	_, err = task.Periodic("periodic_job5_name", sum, 10, 20).EveryHours(2).AtMinuteInHour(20, 30)
+	if err != nil {
+		return
+	}
+
+	// will run every 5 minutes at :20
+	job, err := task.Periodic("periodic_job6_name", sum, 10, 20).EveryMinutes(5).AtSecondInMinute(20)
+	if err != nil {
+		return
+	}
+
+	// task.StartScheduler method will start the scheduler, it will loop to schedule runnable jobs.
+	// you can add more jobs to schedule after call task.StartScheduler.
+	// here, to show more use cases, start scheduler in a separate goroutine
+	go task.StartScheduler()
+
+	// return job's name: periodic_job6_name
+	job.Name()
+
+	// return job's running statistics
+	jobStats := job.Stats()
+	for _, stat := range jobStats {
+		fmt.Println(stat.ToMap())
+	}
+
+	// return jobs's latest scheduled time, return time.Time{} if not scheduled yet.
+	job.GetLatestScheduledTime()
+
+	// return job count in scheduler
+	task.JobCount()
+
+	// handle all job stats
+	allJobStats := task.JobStats()
+	for jobName, jobStats := range allJobStats {
+		fmt.Printf("job %s stat:\n", jobName)
+		for _, stat := range jobStats {
+			fmt.Println(stat.ToMap())
+		}
+	}
+	// remove job by name in scheduler
+	task.RemoveJob(job.Name())
+
+	// remove all jobs in scheduler
+	task.RemoveAllJobs()
+
+	// Stop scheduler after 5 seconds
+	// set argument to false indicates waiting all running jobs finish before return.
+	time.Sleep(5 * time.Second)
+	task.StopScheduler(false)
+}
+
+// Show how to coordinate and schedule multiple job instances with the same name.
+func ExampleCoordinate() {
+	coordinator := task.NewCoordinatorFromRedis("coordinator1", "localhost:6379")
+	// with redis cluster, use:
+	// task.NewCoordinatorFromRedisCluster("coordinator2", []string{"localhost:30000", "localhost:30001"})
+
+	scheduler1 := task.NewScheduler(10)
+	scheduler2 := task.NewScheduler(10)
+
+	name := "coordinate_job"
+	job1 := scheduler1.AddPeriodicJob(name, sum, 1, 2).EverySeconds(2).SetCoordinate(coordinator)
+	job2 := scheduler2.AddPeriodicJob(name, sum, 3, 4).EverySeconds(2).SetCoordinate(coordinator)
+
+	// job1 and job2 will coordinate, only one of them will be scheduled once every 2 seconds
+	go scheduler1.Start()
+	go scheduler2.Start()
+
+	jobStats := job1.Stats()
+	fmt.Println("job1 stats:")
+	for _, stat := range jobStats {
+		fmt.Println(stat.ToMap())
+	}
+
+	jobStats = job2.Stats()
+	fmt.Println("job2 stats:")
+	for _, stat := range jobStats {
+		fmt.Println(stat.ToMap())
+	}
+	// stop schedulers after 10 seconds
+	time.Sleep(10 * time.Second)
+
+	scheduler1.Stop(false)
+	scheduler2.Stop(false)
+}
+
+func sum(a, b int) int {
+	return a + b
+}