From 367ba5310f3e02e618ea3cb9bb1afdf3f6fa753a Mon Sep 17 00:00:00 2001 From: shellus Date: Mon, 28 Aug 2017 20:13:31 +0800 Subject: [PATCH] fix bugs --- LICENSE | 25 + Makefile | 19 + README.md | 143 +- bench/client.cpp | 200 -- bench/server.cpp | 58 - bench_test.go | 216 ++ cluster.go | 1170 +++++++ cluster_test.go | 800 +++++ command.go | 956 ++++++ command_test.go | 60 + commands.go | 2133 ++++++++++++ commands_test.go | 2942 +++++++++++++++++ doc.go | 4 + example_instrumentation_test.go | 59 + example_test.go | 414 +++ export_test.go | 36 + internal/consistenthash/consistenthash.go | 81 + .../consistenthash/consistenthash_test.go | 110 + internal/error.go | 75 + internal/hashtag/hashtag.go | 73 + internal/hashtag/hashtag_test.go | 74 + internal/internal.go | 24 + internal/internal_test.go | 18 + internal/log.go | 15 + internal/once.go | 60 + internal/pool/bench_test.go | 80 + internal/pool/conn.go | 78 + internal/pool/main_test.go | 35 + internal/pool/pool.go | 371 +++ internal/pool/pool_single.go | 55 + internal/pool/pool_sticky.go | 123 + internal/pool/pool_test.go | 241 ++ internal/proto/proto_test.go | 13 + internal/proto/reader.go | 334 ++ internal/proto/reader_test.go | 87 + internal/proto/scan.go | 131 + internal/proto/scan_test.go | 48 + internal/proto/write_buffer.go | 103 + internal/proto/write_buffer_test.go | 63 + internal/safe.go | 11 + internal/unsafe.go | 27 + internal/util.go | 47 + iterator.go | 73 + iterator_test.go | 136 + main_test.go | 355 ++ options.go | 210 ++ options_test.go | 94 + parser.go | 374 +++ pipeline.go | 106 + pipeline_test.go | 80 + pool_test.go | 141 + pubsub.go | 392 +++ pubsub_test.go | 427 +++ race_test.go | 247 ++ redis.go | 458 +++ redis_context.go | 35 + redis_no_context.go | 15 + redis_test.go | 364 ++ result.go | 140 + ring.go | 458 +++ ring_test.go | 193 ++ script.go | 62 + sentinel.go | 335 ++ sentinel_test.go | 88 + src/all.h | 349 -- src/makefile | 47 - src/xAcceptor.cpp | 57 - src/xAcceptor.h | 36 - src/xBuffer.cpp | 38 - src/xBuffer.h | 351 -- src/xCallback.h | 33 - src/xChannel.cpp | 100 - src/xChannel.h | 60 - src/xCluster.cpp | 433 --- src/xCluster.h | 49 - src/xConnector.cpp | 110 - src/xConnector.h | 54 - src/xCrc16.h | 89 - src/xCrc64.cpp | 189 -- src/xCrc64.h | 4 - src/xCurrentThread.h | 20 - src/xEpoll.cpp | 197 -- src/xEpoll.h | 32 - src/xEventLoop.cpp | 196 -- src/xEventLoop.h | 62 - src/xHiredis.cpp | 1489 --------- src/xHiredis.h | 218 -- src/xHttpContext.cpp | 171 - src/xHttpContext.h | 45 - src/xHttpRequest.h | 202 -- src/xHttpResponse.cpp | 33 - src/xHttpResponse.h | 66 - src/xHttpServer.cpp | 85 - src/xHttpServer.h | 30 - src/xLog.cpp | 537 --- src/xLog.h | 336 -- src/xLzf.cpp | 291 -- src/xLzf.h | 12 - src/xLzfd.cpp | 185 -- src/xLzfp.h | 184 -- src/xMain.cpp | 69 - src/xObject.cpp | 719 ---- src/xObject.h | 136 - src/xPosix.h | 104 - src/xRdb.cpp | 1326 -------- src/xRdb.h | 115 - src/xRedis.cpp | 2474 -------------- src/xRedis.h | 190 -- src/xReplication.cpp | 243 -- src/xReplication.h | 49 - src/xSds.cpp | 1031 ------ src/xSds.h | 77 - src/xSentinel.cpp | 98 - src/xSentinel.h | 31 - src/xSession.cpp | 527 --- src/xSession.h | 41 - src/xSocket.cpp | 182 - src/xSocket.h | 34 - src/xTcpClient.cpp | 121 - src/xTcpClient.h | 48 - src/xTcpServer.cpp | 85 - src/xTcpServer.h | 47 - src/xTcpconnection.cpp | 315 -- src/xTcpconnection.h | 85 - src/xThread.cpp | 67 - src/xThread.h | 25 - src/xThreadPool.cpp | 86 - src/xThreadPool.h | 33 - src/xTimer.cpp | 45 - src/xTimer.h | 67 - src/xTimerQueue.cpp | 316 -- src/xTimerQueue.h | 49 - src/xZmalloc.cpp | 389 --- src/xZmalloc.h | 236 -- testdata/redis.conf | 10 + tx.go | 96 + tx_test.go | 151 + universal.go | 133 + universal_test.go | 41 + 139 files changed, 16566 insertions(+), 16110 deletions(-) create mode 100644 LICENSE create mode 100644 Makefile delete mode 100755 bench/client.cpp delete mode 100755 bench/server.cpp create mode 100644 bench_test.go create mode 100644 cluster.go create mode 100644 cluster_test.go create mode 100644 command.go create mode 100644 command_test.go create mode 100644 commands.go create mode 100644 commands_test.go create mode 100644 doc.go create mode 100644 example_instrumentation_test.go create mode 100644 example_test.go create mode 100644 export_test.go create mode 100644 internal/consistenthash/consistenthash.go create mode 100644 internal/consistenthash/consistenthash_test.go create mode 100644 internal/error.go create mode 100644 internal/hashtag/hashtag.go create mode 100644 internal/hashtag/hashtag_test.go create mode 100644 internal/internal.go create mode 100644 internal/internal_test.go create mode 100644 internal/log.go create mode 100644 internal/once.go create mode 100644 internal/pool/bench_test.go create mode 100644 internal/pool/conn.go create mode 100644 internal/pool/main_test.go create mode 100644 internal/pool/pool.go create mode 100644 internal/pool/pool_single.go create mode 100644 internal/pool/pool_sticky.go create mode 100644 internal/pool/pool_test.go create mode 100644 internal/proto/proto_test.go create mode 100644 internal/proto/reader.go create mode 100644 internal/proto/reader_test.go create mode 100644 internal/proto/scan.go create mode 100644 internal/proto/scan_test.go create mode 100644 internal/proto/write_buffer.go create mode 100644 internal/proto/write_buffer_test.go create mode 100644 internal/safe.go create mode 100644 internal/unsafe.go create mode 100644 internal/util.go create mode 100644 iterator.go create mode 100644 iterator_test.go create mode 100644 main_test.go create mode 100644 options.go create mode 100644 options_test.go create mode 100644 parser.go create mode 100644 pipeline.go create mode 100644 pipeline_test.go create mode 100644 pool_test.go create mode 100644 pubsub.go create mode 100644 pubsub_test.go create mode 100644 race_test.go create mode 100644 redis.go create mode 100644 redis_context.go create mode 100644 redis_no_context.go create mode 100644 redis_test.go create mode 100644 result.go create mode 100644 ring.go create mode 100644 ring_test.go create mode 100644 script.go create mode 100644 sentinel.go create mode 100644 sentinel_test.go delete mode 100644 src/all.h delete mode 100755 src/makefile delete mode 100644 src/xAcceptor.cpp delete mode 100644 src/xAcceptor.h delete mode 100644 src/xBuffer.cpp delete mode 100644 src/xBuffer.h delete mode 100644 src/xCallback.h delete mode 100644 src/xChannel.cpp delete mode 100755 src/xChannel.h delete mode 100755 src/xCluster.cpp delete mode 100755 src/xCluster.h delete mode 100644 src/xConnector.cpp delete mode 100644 src/xConnector.h delete mode 100644 src/xCrc16.h delete mode 100644 src/xCrc64.cpp delete mode 100644 src/xCrc64.h delete mode 100644 src/xCurrentThread.h delete mode 100644 src/xEpoll.cpp delete mode 100644 src/xEpoll.h delete mode 100644 src/xEventLoop.cpp delete mode 100644 src/xEventLoop.h delete mode 100644 src/xHiredis.cpp delete mode 100644 src/xHiredis.h delete mode 100644 src/xHttpContext.cpp delete mode 100644 src/xHttpContext.h delete mode 100644 src/xHttpRequest.h delete mode 100644 src/xHttpResponse.cpp delete mode 100644 src/xHttpResponse.h delete mode 100644 src/xHttpServer.cpp delete mode 100644 src/xHttpServer.h delete mode 100755 src/xLog.cpp delete mode 100644 src/xLog.h delete mode 100644 src/xLzf.cpp delete mode 100644 src/xLzf.h delete mode 100644 src/xLzfd.cpp delete mode 100644 src/xLzfp.h delete mode 100755 src/xMain.cpp delete mode 100644 src/xObject.cpp delete mode 100644 src/xObject.h delete mode 100644 src/xPosix.h delete mode 100644 src/xRdb.cpp delete mode 100644 src/xRdb.h delete mode 100755 src/xRedis.cpp delete mode 100755 src/xRedis.h delete mode 100644 src/xReplication.cpp delete mode 100644 src/xReplication.h delete mode 100644 src/xSds.cpp delete mode 100644 src/xSds.h delete mode 100644 src/xSentinel.cpp delete mode 100644 src/xSentinel.h delete mode 100755 src/xSession.cpp delete mode 100755 src/xSession.h delete mode 100644 src/xSocket.cpp delete mode 100644 src/xSocket.h delete mode 100644 src/xTcpClient.cpp delete mode 100644 src/xTcpClient.h delete mode 100644 src/xTcpServer.cpp delete mode 100644 src/xTcpServer.h delete mode 100644 src/xTcpconnection.cpp delete mode 100644 src/xTcpconnection.h delete mode 100644 src/xThread.cpp delete mode 100644 src/xThread.h delete mode 100644 src/xThreadPool.cpp delete mode 100644 src/xThreadPool.h delete mode 100644 src/xTimer.cpp delete mode 100644 src/xTimer.h delete mode 100644 src/xTimerQueue.cpp delete mode 100644 src/xTimerQueue.h delete mode 100644 src/xZmalloc.cpp delete mode 100644 src/xZmalloc.h create mode 100644 testdata/redis.conf create mode 100644 tx.go create mode 100644 tx_test.go create mode 100644 universal.go create mode 100644 universal_test.go diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..298bed9b --- /dev/null +++ b/LICENSE @@ -0,0 +1,25 @@ +Copyright (c) 2013 The github.com/go-redis/redis Authors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..50fdc55a --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +all: testdeps + go test ./... + go test ./... -short -race + go vet + +testdeps: testdata/redis/src/redis-server + +bench: testdeps + go test ./... -test.run=NONE -test.bench=. -test.benchmem + +.PHONY: all test testdeps bench + +testdata/redis: + mkdir -p $@ + wget -qO- https://github.com/antirez/redis/archive/unstable.tar.gz | tar xvz --strip-components=1 -C $@ + +testdata/redis/src/redis-server: testdata/redis + sed -i 's/libjemalloc.a/libjemalloc.a -lrt/g' $ +} + +func ExampleClient() { + err := client.Set("key", "value", 0).Err() + if err != nil { + panic(err) + } + + val, err := client.Get("key").Result() + if err != nil { + panic(err) + } + fmt.Println("key", val) + + val2, err := client.Get("key2").Result() + if err == redis.Nil { + fmt.Println("key2 does not exists") + } else if err != nil { + panic(err) + } else { + fmt.Println("key2", val2) + } + // Output: key value + // key2 does not exists +} +``` + +## Howto + +Please go through [examples](https://godoc.org/github.com/go-redis/redis#pkg-examples) to get an idea how to use this package. + +## Look and feel + +Some corner cases: + + SET key value EX 10 NX + set, err := client.SetNX("key", "value", 10*time.Second).Result() + + SORT list LIMIT 0 2 ASC + vals, err := client.Sort("list", redis.Sort{Offset: 0, Count: 2, Order: "ASC"}).Result() + + ZRANGEBYSCORE zset -inf +inf WITHSCORES LIMIT 0 2 + vals, err := client.ZRangeByScoreWithScores("zset", redis.ZRangeBy{ + Min: "-inf", + Max: "+inf", + Offset: 0, + Count: 2, + }).Result() + + ZINTERSTORE out 2 zset1 zset2 WEIGHTS 2 3 AGGREGATE SUM + vals, err := client.ZInterStore("out", redis.ZStore{Weights: []int64{2, 3}}, "zset1", "zset2").Result() + + EVAL "return {KEYS[1],ARGV[1]}" 1 "key" "hello" + vals, err := client.Eval("return {KEYS[1],ARGV[1]}", []string{"key"}, "hello").Result() + +## Benchmark + +go-redis vs redigo: + +``` +BenchmarkSetGoRedis10Conns64Bytes-4 200000 7621 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns64Bytes-4 200000 7554 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns1KB-4 200000 7697 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns1KB-4 200000 7688 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns10KB-4 200000 9214 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns10KB-4 200000 9181 ns/op 210 B/op 6 allocs/op +BenchmarkSetGoRedis10Conns1MB-4 2000 583242 ns/op 2337 B/op 6 allocs/op +BenchmarkSetGoRedis100Conns1MB-4 2000 583089 ns/op 2338 B/op 6 allocs/op +BenchmarkSetRedigo10Conns64Bytes-4 200000 7576 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns64Bytes-4 200000 7782 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns1KB-4 200000 7958 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns1KB-4 200000 7725 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns10KB-4 100000 18442 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo100Conns10KB-4 100000 18818 ns/op 208 B/op 7 allocs/op +BenchmarkSetRedigo10Conns1MB-4 2000 668829 ns/op 226 B/op 7 allocs/op +BenchmarkSetRedigo100Conns1MB-4 2000 679542 ns/op 226 B/op 7 allocs/op +``` + +Redis Cluster: + +``` +BenchmarkRedisPing-4 200000 6983 ns/op 116 B/op 4 allocs/op +BenchmarkRedisClusterPing-4 100000 11535 ns/op 117 B/op 4 allocs/op +``` + +## See also + +- [Golang PostgreSQL ORM](https://github.com/go-pg/pg) +- [Golang msgpack](https://github.com/vmihailenco/msgpack) +- [Golang message task queue](https://github.com/go-msgqueue/msgqueue) diff --git a/bench/client.cpp b/bench/client.cpp deleted file mode 100755 index 7b769cdd..00000000 --- a/bench/client.cpp +++ /dev/null @@ -1,200 +0,0 @@ -#pragma once -#include "all.h" - -#include "xTcpClient.h" -#include "xTcpconnection.h" -#include "xThreadPool.h" -#include "xCurrentThread.h" -#include "xLog.h" - - -class client; - -class session:noncopyable -{ -public: - session(xEventLoop *loop, - const char *ip,uint16_t port,client *owner) - :cli(loop,nullptr), - ip(ip), - port(port), - owner(owner), - bytesRead(0), - bytesWritten(0), - messagesRead(0) - { - cli.setConnectionCallback(std::bind(&session::connCallBack, this, std::placeholders::_1, std::placeholders::_2)); - cli.setMessageCallback(std::bind(&session::readCallBack, this, std::placeholders::_1, std::placeholders::_2, std::placeholders::_3)); - cli.setConnectionErrorCallBack(std::bind(&session::connErrorCallBack, this)); - } - - void start() - { - cli.connect(ip,port); - } - void stop() - { - cli.disconnect(); - } - - - int64_t getBytesRead() { return bytesRead; } - int64_t getMessagesRead() { return messagesRead; } - -private: - - void connCallBack(const xTcpconnectionPtr& conn, void *data); - void connErrorCallBack() - { - //LOG_WARN<<"tcp connect failure"; - } - - void readCallBack(const xTcpconnectionPtr& conn, xBuffer* buf, void *data) - { - ++messagesRead; - bytesRead += buf->readableBytes(); - bytesWritten += buf->readableBytes(); - conn->send(buf); - buf->retrieveAll(); - } - - xTcpClient cli; - const char *ip; - uint16_t port; - client * owner; - int64_t bytesRead; - int64_t bytesWritten; - int64_t messagesRead; - -}; - -class client:noncopyable -{ -public: - client(xEventLoop *loop,const char *ip,uint16_t port,int blockSize,int sessionCount, - int timeOut,int threadCount) - :loop(loop), - threadPool(loop), - sessionCount(sessionCount), - timeOut(timeOut) - { - loop->runAfter(timeOut,nullptr,false, std::bind(&client::handlerTimeout, this,std::placeholders::_1)); - if(threadCount > 1) - { - threadPool.setThreadNum(threadCount); - } - threadPool.start(); - - for(int i = 0; i < blockSize; i ++) - { - message.push_back(static_cast(i % 128)); - } - - for(int i = 0 ; i < sessionCount; i ++) - { - std::shared_ptr vsession (new session(threadPool.getNextLoop(),ip,port,this)); - vsession->start(); - sessions.push_back(vsession); - numConencted++; - } - } - - void onConnect() - { - if (numConencted == sessionCount) - { - LOG_WARN << "all connected"; - } - } - - - void onDisconnect(const xTcpconnectionPtr& conn) - { - numConencted--; - if (numConencted == 0) - { - LOG_WARN << "all disconnected"; - - int64_t totalBytesRead = 0; - int64_t totalMessagesRead = 0; - for(auto it = sessions.begin(); it != sessions.end(); ++it) - { - totalBytesRead += (*it)->getBytesRead(); - totalMessagesRead += (*it)->getMessagesRead(); - } - LOG_WARN << totalBytesRead << " total bytes read"; - LOG_WARN << totalMessagesRead << " total messages read"; - LOG_WARN << static_cast(totalBytesRead) / static_cast(totalMessagesRead)<< " average message size"; - LOG_WARN << static_cast(totalBytesRead) / (timeOut * 1024 * 1024) << " MiB/s throughput"; - conn->getLoop()->queueInLoop(std::bind(&client::quit, this)); - } - } - - - const std::string &getMessage()const - { - return message; - } -private: - void quit() - { - loop->queueInLoop(std::bind(&xEventLoop::quit,loop)); - - } - - void handlerTimeout(void * data) - { - LOG_WARN << "stop"; - std::for_each(sessions.begin(),sessions.end(),std::mem_fn(&session::stop)); - } - - xEventLoop *loop; - xThreadPool threadPool; - int sessionCount; - int timeOut; - std::vector> sessions; - std::string message; - std::atomic numConencted; -}; - - -void session::connCallBack(const xTcpconnectionPtr & conn,void * data) -{ - if (conn->connected()) - { - conn->send(owner->getMessage()); - owner->onConnect(); - } - else - { - owner->onDisconnect(conn); - } -} - - -int main(int argc,char * argv[]) -{ - if (argc != 7) - { - fprintf(stderr, "Usage: client "); - fprintf(stderr, "