From 795dd24be0c503e740a311c38f065e357b0217e5 Mon Sep 17 00:00:00 2001 From: halalala222 <1741196223@qq.com> Date: Mon, 12 Aug 2024 12:11:43 +0800 Subject: [PATCH 1/6] feat: update db add success info log & error log --- db/engine/db.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/db/engine/db.go b/db/engine/db.go index ee8a0da..ad00286 100644 --- a/db/engine/db.go +++ b/db/engine/db.go @@ -5,12 +5,6 @@ package engine import ( "fmt" - "github.com/ByteStorage/FlyDB/config" - data2 "github.com/ByteStorage/FlyDB/db/data" - "github.com/ByteStorage/FlyDB/db/index" - "github.com/ByteStorage/FlyDB/lib/backup" - "github.com/ByteStorage/FlyDB/lib/const" - "go.uber.org/zap" "io" "os" "path/filepath" @@ -18,6 +12,15 @@ import ( "strconv" "strings" "sync" + + "go.uber.org/zap" + + "github.com/ByteStorage/FlyDB/config" + data2 "github.com/ByteStorage/FlyDB/db/data" + "github.com/ByteStorage/FlyDB/db/index" + "github.com/ByteStorage/FlyDB/lib/backup" + "github.com/ByteStorage/FlyDB/lib/const" + _ "github.com/ByteStorage/FlyDB/lib/logger" ) // DB represents a FlyDB database instance, @@ -147,7 +150,6 @@ func (db *DB) Sync() error { // Put write a key-value pair to db, and the key must be not empty func (db *DB) Put(key []byte, value []byte) error { - zap.L().Info("put", zap.ByteString("key", key), zap.ByteString("value", value)) // check key if len(key) == 0 { return _const.ErrKeyIsEmpty @@ -163,6 +165,7 @@ func (db *DB) Put(key []byte, value []byte) error { // append log record pos, err := db.appendLogRecordWithLock(logRecord) if err != nil { + zap.L().Error("put error", zap.Error(err), zap.Any("operation", NewPutOperation(key, value))) return err } @@ -171,6 +174,7 @@ func (db *DB) Put(key []byte, value []byte) error { return _const.ErrIndexUpdateFailed } + zap.L().Info("put success", zap.Any("operation", NewPutOperation(key, value))) return nil } @@ -348,8 +352,6 @@ func (db *DB) getValueByPosition(logRecordPst *data2.LogRecordPst) ([]byte, erro // Delete data according to the key func (db *DB) Delete(key []byte) error { - zap.L().Info("delete", zap.ByteString("key", key)) - // Determine the validity of the key if len(key) == 0 { return _const.ErrKeyIsEmpty @@ -369,6 +371,7 @@ func (db *DB) Delete(key []byte) error { // Write to the data file _, err := db.appendLogRecordWithLock(logRecord) if err != nil { + zap.L().Error("delete error", zap.Error(err), zap.Any("operation", NewDeleteOperation(key))) return err } @@ -377,6 +380,7 @@ func (db *DB) Delete(key []byte) error { if !ok { return _const.ErrIndexUpdateFailed } + zap.L().Info("delete success", zap.Any("operation", NewDeleteOperation(key))) return nil } From e6096996dc399377edd3452226b15a5a999a57fb Mon Sep 17 00:00:00 2001 From: halalala222 <1741196223@qq.com> Date: Mon, 12 Aug 2024 12:12:48 +0800 Subject: [PATCH 2/6] feat: support reStore & reExecute api --- db/engine/operation.go | 395 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 395 insertions(+) create mode 100644 db/engine/operation.go diff --git a/db/engine/operation.go b/db/engine/operation.go new file mode 100644 index 0000000..70cc377 --- /dev/null +++ b/db/engine/operation.go @@ -0,0 +1,395 @@ +package engine + +import ( + "bufio" + "encoding/json" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "github.com/ByteStorage/FlyDB/lib/logger" +) + +const ( + PutOperationName = "put" + DelOperationName = "delete" + LogConsoleEncoder = "console" + engineDBLog = "engine/db.go" + logMetaCount = 5 + logTimeLayout = "2006-01-02T15:04:05.000Z0700" + infoLogLevel = "INFO" + errorLogLevel = "ERROR" +) + +var ( + ErrLogIsEmpty = errors.New("log is empty") + ErrLogMetaIsNotEnough = errors.New("log meta count is not enough") + ErrLogNotContainOperationField = errors.New("log not contain operation field") + ErrLogUnSupportOperation = errors.New("unsupported operation") + ErrNotSupportEncoder = errors.New("not support encoder") +) + +var ( + supportEncoder = []string{LogConsoleEncoder} +) + +type Operation struct { + Name string `json:"name"` + Key string `json:"key"` + Value string `json:"value"` +} + +func NewPutOperation(key, value []byte) *Operation { + return &Operation{ + Name: PutOperationName, + Key: string(key), + Value: string(value), + } +} + +func NewDeleteOperation(key []byte) *Operation { + return &Operation{ + Name: DelOperationName, + Key: string(key), + } +} + +type OperationLogHandlerOption func(*OperationLogHandler) + +func WithLogConsoleEncoder() OperationLogHandlerOption { + return func(o *OperationLogHandler) { + o.logEncoder = LogConsoleEncoder + } +} + +func WithDB(db *DB) OperationLogHandlerOption { + return func(o *OperationLogHandler) { + o.db = db + } +} + +type OperationLogHandler struct { + logEncoder string + logFilePath string + logLevel string + db *DB + logLinesChan chan string + operationChan chan *Operation + errorChan chan error +} + +func defaultOperationLogHandler() *OperationLogHandler { + return &OperationLogHandler{ + logLinesChan: make(chan string, 1024), + operationChan: make(chan *Operation, 1024), + errorChan: make(chan error, 1024), + } +} + +func NewOperationLogHandler(optionList ...OperationLogHandlerOption) *OperationLogHandler { + handler := defaultOperationLogHandler() + + for _, option := range optionList { + option(handler) + } + + return handler +} + +func (o *OperationLogHandler) readLog(logFilePath string) error { + var ( + file *os.File + err error + scanner *bufio.Scanner + ) + + if file, err = os.Open(logFilePath); err != nil { + return err + } + + defer file.Close() + + scanner = bufio.NewScanner(file) + + for scanner.Scan() { + o.logLinesChan <- scanner.Text() + } + + return scanner.Err() +} + +type logMeta struct { + logTime time.Time + logLevel string + file string + Operation *Operation `json:"operation"` +} + +type operationErr struct { + log string + err error + op *Operation +} + +func (o *operationErr) Error() string { + operationErrStringBuilder := strings.Builder{} + if len(o.log) > 0 { + operationErrStringBuilder.WriteString("log: ") + operationErrStringBuilder.WriteString(o.log) + operationErrStringBuilder.WriteString("\t") + } + if o.err != nil { + operationErrStringBuilder.WriteString("err: ") + operationErrStringBuilder.WriteString(o.err.Error()) + operationErrStringBuilder.WriteString("\t") + } + if o.op != nil { + operationErrStringBuilder.WriteString("operation: ") + operationErrStringBuilder.WriteString(fmt.Sprintf("%+v", o.op)) + } + + return operationErrStringBuilder.String() +} + +var _ error = &operationErr{} + +func (o *OperationLogHandler) parseLogTime(logTime string) (time.Time, error) { + var ( + parsedTime time.Time + err error + ) + + if parsedTime, err = time.Parse(logTimeLayout, logTime); err != nil { + return time.Time{}, err + } + + return parsedTime, nil +} + +// decodeLogConsoleMeta decode log from console encoder +// 2024-08-11T20:23:05.599+0800 INFO engine/db.go:68 open db {"options": {"DirPath":"./data","DataFileSize":268435456,"SyncWrite":false,"IndexType":2,"FIOType":3}} +// time: 2024-08-11T20:23:05.599+0800 +// level: INFO +// file: engine/db.go:68 +// message: open db +// fields: {"options": {"DirPath":"./data","DataFileSize":268435456,"SyncWrite":false,"IndexType":2,"FIOType":3}} +func (o *OperationLogHandler) decodeLogConsoleMeta(log string) (*logMeta, error) { + if len(log) == 0 { + return nil, &operationErr{ + err: ErrLogIsEmpty, + } + } + + var ( + logSplit = strings.Split(log, "\t") + logMetaData = &logMeta{} + err error + ) + + if len(logSplit) < logMetaCount { + return nil, &operationErr{ + log: log, + err: ErrLogMetaIsNotEnough, + } + } + + if logMetaData.logTime, err = o.parseLogTime(logSplit[0]); err != nil { + return nil, &operationErr{ + log: log, + err: err, + } + } + + logMetaData.logLevel = logSplit[1] + logMetaData.file = logSplit[2] + if err = json.Unmarshal([]byte(logSplit[len(logSplit)-1]), &logMetaData); err != nil { + return nil, &operationErr{ + log: log, + err: err, + } + } + + if logMetaData.Operation == nil { + return nil, &operationErr{ + log: log, + err: ErrLogNotContainOperationField, + } + } + + return logMetaData, nil +} + +func (o *OperationLogHandler) checkLogConsoleMeta(logMetaData *logMeta, start time.Time, end time.Time) bool { + if logMetaData.logTime.Before(start) || logMetaData.logTime.After(end) { + return false + } + + if !strings.Contains(logMetaData.file, engineDBLog) { + return false + } + + if logMetaData.logLevel != o.logLevel { + return false + } + + return true +} + +func (o *OperationLogHandler) decodeLogConsoleEncode(start time.Time, end time.Time) { + for line := range o.logLinesChan { + var ( + logMetaData *logMeta + err error + ) + + if logMetaData, err = o.decodeLogConsoleMeta(line); err != nil { + o.errorChan <- err + continue + } + + if !o.checkLogConsoleMeta(logMetaData, start, end) { + continue + } + + o.operationChan <- logMetaData.Operation + } +} + +func (o *OperationLogHandler) restoreOperation() { + for operation := range o.operationChan { + var ( + err error + ) + switch operation.Name { + case PutOperationName: + if err = o.db.Put([]byte(operation.Key), []byte(operation.Value)); err != nil { + o.errorChan <- &operationErr{ + err: err, + op: operation, + } + } + case DelOperationName: + if err = o.db.Delete([]byte(operation.Key)); err != nil { + o.errorChan <- &operationErr{ + err: err, + op: operation, + } + } + default: + o.errorChan <- &operationErr{ + err: ErrLogUnSupportOperation, + op: operation, + } + } + } +} + +func (o *OperationLogHandler) combinedError() error { + var ( + combinedErr error + ) + + for err := range o.errorChan { + if combinedErr == nil { + combinedErr = fmt.Errorf("restore operation error") + } + combinedErr = fmt.Errorf("%w\n%v", combinedErr, err) + } + + return combinedErr +} + +func (o *OperationLogHandler) checkEncoderSupport() bool { + for _, encoder := range supportEncoder { + if o.logEncoder == encoder { + return true + } + } + + return false +} + +func (o *OperationLogHandler) restoreLogOperation(start time.Time, end time.Time) error { + var ( + waitGroup = &sync.WaitGroup{} + ) + + if !o.checkEncoderSupport() { + return ErrNotSupportEncoder + } + + waitGroup.Add(3) + + go func() { + defer waitGroup.Done() + if err := o.readLog(o.logFilePath); err != nil { + o.errorChan <- err + } + close(o.logLinesChan) + }() + + go func() { + defer waitGroup.Done() + o.decodeLogConsoleEncode(start, end) + close(o.operationChan) + }() + + go func() { + defer waitGroup.Done() + o.restoreOperation() + }() + + go func() { + waitGroup.Wait() + close(o.errorChan) + }() + + return o.combinedError() +} + +// RestoreWithTime will restore operation log with time range [start, end] +func (o *OperationLogHandler) RestoreWithTime(start time.Time, end time.Time) error { + o.logFilePath = logger.LogLocation + o.logLevel = infoLogLevel + return o.restoreLogOperation(start, end) +} + +// RestoreAfterStart will restore operation log after start time +func (o *OperationLogHandler) RestoreAfterStart(start time.Time) error { + return o.RestoreWithTime(start, time.Now().Add(time.Hour*24*365)) +} + +// RestoreBeforeEnd will restore operation log before end time +func (o *OperationLogHandler) RestoreBeforeEnd(end time.Time) error { + return o.RestoreWithTime(time.Time{}, end) +} + +// Restore will restore operation log with all time range +func (o *OperationLogHandler) Restore() error { + return o.RestoreWithTime(time.Time{}, time.Now().Add(time.Hour*24*365)) +} + +// ReExecuteWithTime re-execute operation log with time range [start, end] +func (o *OperationLogHandler) ReExecuteWithTime(start time.Time, end time.Time) error { + o.logFilePath = logger.ErrorLogLocation + o.logLevel = errorLogLevel + return o.restoreLogOperation(start, end) +} + +// ReExecuteAfterStart re-execute operation log after start time +func (o *OperationLogHandler) ReExecuteAfterStart(start time.Time) error { + return o.ReExecuteWithTime(start, time.Now().Add(time.Hour*24*365)) +} + +// ReExecuteBeforeEnd re-execute operation log before end time +func (o *OperationLogHandler) ReExecuteBeforeEnd(end time.Time) error { + return o.ReExecuteWithTime(time.Time{}, end) +} + +// ReExecute re-execute operation log with all time range +func (o *OperationLogHandler) ReExecute() error { + return o.ReExecuteWithTime(time.Time{}, time.Now().Add(time.Hour*24*365)) +} From 3c5a96db9222ca60a70a194ac3c93fc2cadf54cd Mon Sep 17 00:00:00 2001 From: halalala222 <1741196223@qq.com> Date: Mon, 12 Aug 2024 12:12:59 +0800 Subject: [PATCH 3/6] feat: add reStore & reExecute api unit test --- db/engine/operation_test.go | 467 ++++++++++++++++++++++++++++++++++++ 1 file changed, 467 insertions(+) create mode 100644 db/engine/operation_test.go diff --git a/db/engine/operation_test.go b/db/engine/operation_test.go new file mode 100644 index 0000000..02eb326 --- /dev/null +++ b/db/engine/operation_test.go @@ -0,0 +1,467 @@ +package engine + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/ByteStorage/FlyDB/config" + "github.com/ByteStorage/FlyDB/lib/logger" +) + +func Test_parseLogTime(t *testing.T) { + operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder()) + + testcase := []struct { + name string + timeString string + expectTimeYear int + expectTimeMonth int + expectTimeHour int + expectTimeMin int + expectTimeSecond int + expectErr bool + }{ + { + name: "parse time", + timeString: "2021-08-01T15:04:05.999+0800", + expectTimeYear: 2021, + expectTimeMonth: 8, + expectTimeHour: 15, + expectTimeMin: 4, + expectTimeSecond: 5, + expectErr: false, + }, + { + name: "parse time", + timeString: "2024-08-11T20:23:07.877+0800", + expectTimeYear: 2024, + expectTimeMonth: 8, + expectTimeHour: 20, + expectTimeMin: 23, + expectTimeSecond: 7, + expectErr: false, + }, + { + name: "bad time", + timeString: "11-08-2024 20:23:05", + expectErr: true, + }, + { + name: "empty time", + timeString: "", + expectErr: true, + }, + } + + for _, testcaseData := range testcase { + t.Run(testcaseData.name, func(t *testing.T) { + parsedTime, err := operationLogHandler.parseLogTime(testcaseData.timeString) + assert.Equal(t, testcaseData.expectErr, err != nil) + if !testcaseData.expectErr { + assert.Equal(t, testcaseData.expectTimeYear, parsedTime.Year()) + assert.Equal(t, testcaseData.expectTimeMonth, int(parsedTime.Month())) + assert.Equal(t, testcaseData.expectTimeHour, parsedTime.Hour()) + assert.Equal(t, testcaseData.expectTimeMin, parsedTime.Minute()) + assert.Equal(t, testcaseData.expectTimeSecond, parsedTime.Second()) + } + }) + } +} + +func Test_decodeLogConsoleMeta(t *testing.T) { + operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder()) + + testcase := []struct { + name string + log string + expectErr bool + expectLogMeta *logMeta + }{ + { + name: "decode put log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + expectLogMeta: &logMeta{ + logLevel: "INFO", + file: "engine/db.go:171", + Operation: &Operation{ + Name: "put", + Key: "test", + Value: "test", + }, + }, + }, + { + name: "decode delete log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tdelete error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + "\"operation\": {\"name\":\"delete\",\"key\":\"test\"}}\n", + expectLogMeta: &logMeta{ + logLevel: "INFO", + file: "engine/db.go:171", + Operation: &Operation{ + Name: "delete", + Key: "test", + }, + }, + expectErr: false, + }, + { + name: "empty log", + log: "", + expectErr: true, + expectLogMeta: nil, + }, + { + name: "bad log log meta is not enough", + log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db", + expectErr: true, + expectLogMeta: nil, + }, + { + name: "bad log without operation", + log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db {\"options\": {\"DirPath\":\"." + + "/data\",\"DataFileSize\":268435456,\"SyncWrite\":false,\"IndexType\":2,\"FIOType\":3}}", + expectErr: true, + expectLogMeta: nil, + }, + { + name: "bad log parse time error", + log: "15:04:05.999+0800\tINFO\tengine/db.go:68 open db {\"options\": {\"DirPath\":\".", + expectErr: true, + expectLogMeta: nil, + }, + { + name: "bad log log field unmarshal error", + log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db errorField", + expectErr: true, + expectLogMeta: nil, + }, + } + + for _, testcaseData := range testcase { + t.Run(testcaseData.name, func(t *testing.T) { + logMetaData, err := operationLogHandler.decodeLogConsoleMeta(testcaseData.log) + assert.Equal(t, testcaseData.expectErr, err != nil) + if !testcaseData.expectErr { + assert.Equal(t, testcaseData.expectLogMeta.logLevel, logMetaData.logLevel) + assert.Equal(t, testcaseData.expectLogMeta.file, logMetaData.file) + assert.Equal(t, testcaseData.expectLogMeta.Operation.Name, logMetaData.Operation.Name) + assert.Equal(t, testcaseData.expectLogMeta.Operation.Key, logMetaData.Operation.Key) + assert.Equal(t, testcaseData.expectLogMeta.Operation.Value, logMetaData.Operation.Value) + } else { + assert.Nil(t, logMetaData) + } + }) + } +} + +func Test_readLog(t *testing.T) { + mockLogs := []string{ + "log1\n", + "log2\n", + "log3\n", + "log4\n", + "log5\n", + "log6\n", + } + + tmpFile, err := ioutil.TempFile("", "test") + + if err != nil { + t.Fatal(err) + } + + defer func() { + if err = os.Remove(tmpFile.Name()); err != nil { + t.Fatal(err) + } + }() + + for _, log := range mockLogs { + if _, err = tmpFile.WriteString(log); err != nil { + t.Fatal(err) + } + } + + if _, err = tmpFile.Seek(0, 0); err != nil { + t.Fatal(err) + } + + operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder()) + + if err = operationLogHandler.readLog(tmpFile.Name()); err != nil { + t.Fatal(err) + } + + assert.Equal(t, len(mockLogs), len(operationLogHandler.logLinesChan)) + + for _, log := range mockLogs { + assert.Equal(t, log, <-operationLogHandler.logLinesChan+"\n") + } +} + +func Test_readLogError(t *testing.T) { + operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder()) + + err := operationLogHandler.readLog("not_exist_file") + + assert.Error(t, err) + assert.Equal(t, 0, len(operationLogHandler.logLinesChan)) +} + +func TestDecodeLogConsoleEncode(t *testing.T) { + operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder()) + operationLogHandler.logLevel = infoLogLevel + expectOperationList := make([]*Operation, 0) + + testcase := []struct { + name string + log string + expectErr bool + expectOperation *Operation + }{ + { + name: "decode put log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + expectOperation: &Operation{ + Name: "put", + Key: "test", + Value: "test", + }, + }, + { + name: "decode delete log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tdelete error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + "\"operation\": {\"name\":\"delete\",\"key\":\"test\"}}\n", + expectOperation: &Operation{ + Name: "delete", + Key: "test", + }, + }, + { + name: "decode put log file name not engin/db.go", + log: "2024-08-11T20:23:07.877+0800\tINFO\tnotEngine/db." + + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + expectOperation: nil, + }, + { + name: "decode time is before start", + log: "1999-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + expectOperation: nil, + }, + { + name: "decode time is after end", + log: "2800-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + expectOperation: nil, + }, + { + name: "decode log console error", + log: "", + expectOperation: nil, + }, + } + + for _, testcaseData := range testcase { + if testcaseData.expectOperation != nil { + expectOperationList = append(expectOperationList, testcaseData.expectOperation) + } + + operationLogHandler.logLinesChan <- testcaseData.log + } + close(operationLogHandler.logLinesChan) + + location, _ := time.LoadLocation("Asia/Shanghai") + operationLogHandler.decodeLogConsoleEncode(time.Date(2024, 8, 11, 20, 20, 0, 0, location), time.Now().Add(time.Hour)) + close(operationLogHandler.operationChan) + assert.Equal(t, len(expectOperationList), len(operationLogHandler.operationChan)) + + for getOperation := range operationLogHandler.operationChan { + assert.Equal(t, expectOperationList[0].Name, getOperation.Name) + assert.Equal(t, expectOperationList[0].Key, getOperation.Key) + assert.Equal(t, expectOperationList[0].Value, getOperation.Value) + expectOperationList = expectOperationList[1:] + } +} + +func TestRestoreOperation(t *testing.T) { + opts := config.DefaultOptions + dir, _ := os.MkdirTemp("./", "flydb") + opts.DirPath = dir + db, err := NewDB(opts) + if err != nil { + t.Fatal(err) + } + defer destroyDB(db) + + operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder(), WithDB(db)) + + testcases := []struct { + name string + operation *Operation + }{ + { + name: "put operation", + operation: &Operation{ + Name: "put", + Key: "put_key", + Value: "put_value", + }, + }, + { + name: "delete operation", + operation: &Operation{ + Name: "delete", + Key: "put_key", + }, + }, + { + name: "unknown operation", + operation: &Operation{ + Name: "unknown", + Key: "put_key", + }, + }, + } + + for _, testcaseData := range testcases { + operationLogHandler.operationChan <- testcaseData.operation + } + + close(operationLogHandler.operationChan) + + operationLogHandler.restoreOperation() + + value, err := db.Get([]byte(testcases[0].operation.Key)) + assert.Error(t, err) + assert.Nil(t, value) +} + +func TestOperationLogHandler_Restore(t *testing.T) { + opts := config.DefaultOptions + dir, _ := os.MkdirTemp("./", "flydb") + opts.DirPath = dir + db, err := NewDB(opts) + if err != nil { + t.Fatal(err) + } + defer destroyDB(db) + defer func() { + if err = os.RemoveAll(filepath.Dir(logger.LogLocation)); err != nil { + t.Fatal(err) + } + }() + + if err = db.Put([]byte("need_to_remove"), []byte("need to remove")); err != nil { + t.Fatal(err) + } + + operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder(), WithDB(db)) + + testcases := []struct { + name string + log string + }{ + { + name: "success put log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test-success\",\"value\":\"test\"}}\n", + }, + { + name: "success delete log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tdelete mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"delete\",\"key\":\"test-success\"}}\n", + }, + { + name: "error log level log", + log: "2024-08-11T20:23:07.877+0800\tERROR\tengine/db." + + "go:171\tdelete mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"delete\",\"key\":\"test\"}}\n", + }, + { + name: "time is before start", + log: "1999-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + }, + { + name: "time is after end", + log: "2800-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + }, + { + name: "file name is not engine/db.go", + log: "2024-08-11T20:23:07.877+0800\tINFO\tnotEngine/db." + + "go:171\tput mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + }, + { + name: "parse time error", + log: "15:04:05.999+0800\tINFO\tengine/db.go:68 error parse time" + + "\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", + }, + { + name: "log field length is not enough", + log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db\n", + }, + { + name: "success put new log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tput mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"put\",\"key\":\"new_key\",\"value\":\"test\"}}\n", + }, + { + name: "delete need to remove log", + log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + + "go:171\tdelete mock\t{\"mock\": \"mock log\", " + + "\"operation\": {\"name\":\"delete\",\"key\":\"need_to_remove\"}}\n", + }, + } + + logFile, err := os.OpenFile(logger.LogLocation, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) + if err != nil { + t.Fatal(err) + } + + for _, testcaseData := range testcases { + if _, err = logFile.WriteString(testcaseData.log); err != nil { + t.Fatal(err) + } + } + + location, _ := time.LoadLocation("Asia/Shanghai") + err = operationLogHandler.RestoreWithTime(time.Date(2024, 8, 11, 20, 20, 0, 0, location), time.Date(2024, 8, 11, 23, 20, + 0, 0, location)) + assert.Error(t, err) + t.Log(err) + + value, err := db.Get([]byte("test")) + assert.Error(t, err) + assert.Nil(t, value) + + value, err = db.Get([]byte("new_key")) + assert.NoError(t, err) + assert.Equal(t, "test", string(value)) + + value, err = db.Get([]byte("need_to_remove")) + assert.Error(t, err) + assert.Nil(t, value) +} From 191ea4bba42d5018aa92d51f3a725cc323b4167f Mon Sep 17 00:00:00 2001 From: halalala222 <1741196223@qq.com> Date: Mon, 12 Aug 2024 14:48:00 +0800 Subject: [PATCH 4/6] style: fix gofmt conflict between go1.18 and go1.20 --- db/engine/operation_test.go | 325 ++++++++++++++++++++---------------- 1 file changed, 181 insertions(+), 144 deletions(-) diff --git a/db/engine/operation_test.go b/db/engine/operation_test.go index 02eb326..f2274ab 100644 --- a/db/engine/operation_test.go +++ b/db/engine/operation_test.go @@ -73,21 +73,33 @@ func Test_parseLogTime(t *testing.T) { } } +type decodeLogConsoleMetaTestCase struct { + name string + log string + expectErr bool + expectLogMeta *logMeta +} + +func newDecodeLogConsoleMetaTestCase(name string, log string, expectErr bool, expectLogMeat *logMeta) *decodeLogConsoleMetaTestCase { + return &decodeLogConsoleMetaTestCase{ + name: name, + log: log, + expectErr: expectErr, + expectLogMeta: expectLogMeat, + } +} + func Test_decodeLogConsoleMeta(t *testing.T) { operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder()) - testcase := []struct { - name string - log string - expectErr bool - expectLogMeta *logMeta - }{ - { - name: "decode put log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + testcase := []*decodeLogConsoleMetaTestCase{ + newDecodeLogConsoleMetaTestCase( + "decode put log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - expectLogMeta: &logMeta{ + false, + &logMeta{ logLevel: "INFO", file: "engine/db.go:171", Operation: &Operation{ @@ -96,13 +108,14 @@ func Test_decodeLogConsoleMeta(t *testing.T) { Value: "test", }, }, - }, - { - name: "decode delete log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tdelete error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + ), + newDecodeLogConsoleMetaTestCase( + "decode delete log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tdelete error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", "+ "\"operation\": {\"name\":\"delete\",\"key\":\"test\"}}\n", - expectLogMeta: &logMeta{ + false, + &logMeta{ logLevel: "INFO", file: "engine/db.go:171", Operation: &Operation{ @@ -110,39 +123,38 @@ func Test_decodeLogConsoleMeta(t *testing.T) { Key: "test", }, }, - expectErr: false, - }, - { - name: "empty log", - log: "", - expectErr: true, - expectLogMeta: nil, - }, - { - name: "bad log log meta is not enough", - log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db", - expectErr: true, - expectLogMeta: nil, - }, - { - name: "bad log without operation", - log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db {\"options\": {\"DirPath\":\"." + - "/data\",\"DataFileSize\":268435456,\"SyncWrite\":false,\"IndexType\":2,\"FIOType\":3}}", - expectErr: true, - expectLogMeta: nil, - }, - { - name: "bad log parse time error", - log: "15:04:05.999+0800\tINFO\tengine/db.go:68 open db {\"options\": {\"DirPath\":\".", - expectErr: true, - expectLogMeta: nil, - }, - { - name: "bad log log field unmarshal error", - log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db errorField", - expectErr: true, - expectLogMeta: nil, - }, + ), + newDecodeLogConsoleMetaTestCase( + "empty log", + "", + true, + nil, + ), + newDecodeLogConsoleMetaTestCase( + "bad log log field length is not enough", + "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db\n", + true, + nil, + ), + newDecodeLogConsoleMetaTestCase( + "bad log without operation", + "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db {\"options\": {\"DirPath\":\"."+ + "/data\",\"DataFileSize\":268435456,\"SyncWrite\":false,\"IndexType\":2,\"FIOType\":3}}\n", + true, + nil, + ), + newDecodeLogConsoleMetaTestCase( + "bad log parse time error", + "15:04:05.999+0800\tINFO\tengine/db.go:68 open db {\"options\": {\"DirPath\":\".\n", + true, + nil, + ), + newDecodeLogConsoleMetaTestCase( + "bad log log field unmarshal error", + "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db errorField", + true, + nil, + ), } for _, testcaseData := range testcase { @@ -216,64 +228,81 @@ func Test_readLogError(t *testing.T) { assert.Equal(t, 0, len(operationLogHandler.logLinesChan)) } +type testDecodeLogConsoleEncodeTestcase struct { + name string + log string + expectErr bool + expectOperation *Operation +} + +func newTestDecodeLogConsoleEncodeTestcase(name string, log string, expectErr bool, expectOperation *Operation) *testDecodeLogConsoleEncodeTestcase { + return &testDecodeLogConsoleEncodeTestcase{ + name: name, + log: log, + expectErr: expectErr, + expectOperation: expectOperation, + } +} + func TestDecodeLogConsoleEncode(t *testing.T) { operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder()) operationLogHandler.logLevel = infoLogLevel expectOperationList := make([]*Operation, 0) - testcase := []struct { - name string - log string - expectErr bool - expectOperation *Operation - }{ - { - name: "decode put log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + testcase := []*testDecodeLogConsoleEncodeTestcase{ + newTestDecodeLogConsoleEncodeTestcase( + "decode put log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - expectOperation: &Operation{ + false, + &Operation{ Name: "put", Key: "test", Value: "test", }, - }, - { - name: "decode delete log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tdelete error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + ), + newTestDecodeLogConsoleEncodeTestcase( + "decode delete log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tdelete error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", "+ "\"operation\": {\"name\":\"delete\",\"key\":\"test\"}}\n", - expectOperation: &Operation{ + false, + &Operation{ Name: "delete", Key: "test", }, - }, - { - name: "decode put log file name not engin/db.go", - log: "2024-08-11T20:23:07.877+0800\tINFO\tnotEngine/db." + - "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + ), + newTestDecodeLogConsoleEncodeTestcase( + "decode put log file name not engin/db.go", + "2024-08-11T20:23:07.877+0800\tINFO\tnotEngine/db."+ + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - expectOperation: nil, - }, - { - name: "decode time is before start", - log: "1999-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + false, + nil, + ), + newTestDecodeLogConsoleEncodeTestcase( + "decode put log time is before start", + "1999-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - expectOperation: nil, - }, - { - name: "decode time is after end", - log: "2800-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", " + + false, + nil, + ), + newTestDecodeLogConsoleEncodeTestcase( + "decode put log time is after end", + "2800-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput error\t{\"error\": \"truncate data\\\\000000000.data: Access is denied.\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - expectOperation: nil, - }, - { - name: "decode log console error", - log: "", - expectOperation: nil, - }, + false, + nil, + ), + newTestDecodeLogConsoleEncodeTestcase( + "decode put log parse time error", + "", + false, + nil, + ), } for _, testcaseData := range testcase { @@ -351,6 +380,18 @@ func TestRestoreOperation(t *testing.T) { assert.Nil(t, value) } +type testOperationLogHandlerRestoreTestCase struct { + name string + log string +} + +func newTestOperationLogHandlerRestoreTestCase(name string, log string) *testOperationLogHandlerRestoreTestCase { + return &testOperationLogHandlerRestoreTestCase{ + name: name, + log: log, + } +} + func TestOperationLogHandler_Restore(t *testing.T) { opts := config.DefaultOptions dir, _ := os.MkdirTemp("./", "flydb") @@ -372,68 +413,64 @@ func TestOperationLogHandler_Restore(t *testing.T) { operationLogHandler := NewOperationLogHandler(WithLogConsoleEncoder(), WithDB(db)) - testcases := []struct { - name string - log string - }{ - { - name: "success put log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput mock\t{\"mock\": \"mock log\", " + + testcases := []*testOperationLogHandlerRestoreTestCase{ + newTestOperationLogHandlerRestoreTestCase( + "success put log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test-success\",\"value\":\"test\"}}\n", - }, - { - name: "success delete log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tdelete mock\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "success delete log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tdelete mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"delete\",\"key\":\"test-success\"}}\n", - }, - { - name: "error log level log", - log: "2024-08-11T20:23:07.877+0800\tERROR\tengine/db." + - "go:171\tdelete mock\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "error log level log", + "2024-08-11T20:23:07.877+0800\tERROR\tengine/db."+ + "go:171\tdelete mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"delete\",\"key\":\"test\"}}\n", - }, - { - name: "time is before start", - log: "1999-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput mock\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "time is before start", + "1999-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - }, - { - name: "time is after end", - log: "2800-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput mock\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "time is after end", + "2800-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - }, - { - name: "file name is not engine/db.go", - log: "2024-08-11T20:23:07.877+0800\tINFO\tnotEngine/db." + - "go:171\tput mock\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "file name is not engine/db.go", + "2024-08-11T20:23:07.877+0800\tINFO\tnotEngine/db."+ + "go:171\tput mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - }, - { - name: "parse time error", - log: "15:04:05.999+0800\tINFO\tengine/db.go:68 error parse time" + - "\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "parse time error", + "15:04:05.999+0800\tINFO\tengine/db.go:68 error parse time\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"test\",\"value\":\"test\"}}\n", - }, - { - name: "log field length is not enough", - log: "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db\n", - }, - { - name: "success put new log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tput mock\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "log field length is not enough", + "2021-08-01T15:04:05.999+0800\tINFO\tengine/db.go:68 open db\n", + ), + newTestOperationLogHandlerRestoreTestCase( + "success put new log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tput mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"put\",\"key\":\"new_key\",\"value\":\"test\"}}\n", - }, - { - name: "delete need to remove log", - log: "2024-08-11T20:23:07.877+0800\tINFO\tengine/db." + - "go:171\tdelete mock\t{\"mock\": \"mock log\", " + + ), + newTestOperationLogHandlerRestoreTestCase( + "delete need to remove log", + "2024-08-11T20:23:07.877+0800\tINFO\tengine/db."+ + "go:171\tdelete mock\t{\"mock\": \"mock log\", "+ "\"operation\": {\"name\":\"delete\",\"key\":\"need_to_remove\"}}\n", - }, + ), } logFile, err := os.OpenFile(logger.LogLocation, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666) From 732ced0e620b51560e24989d5337111febb96de2 Mon Sep 17 00:00:00 2001 From: halalala222 <1741196223@qq.com> Date: Mon, 12 Aug 2024 15:16:03 +0800 Subject: [PATCH 5/6] fix: remove log files to avoid mutual interference. --- db/engine/operation_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/db/engine/operation_test.go b/db/engine/operation_test.go index f2274ab..1d7727b 100644 --- a/db/engine/operation_test.go +++ b/db/engine/operation_test.go @@ -401,6 +401,11 @@ func TestOperationLogHandler_Restore(t *testing.T) { t.Fatal(err) } defer destroyDB(db) + if _, err = os.Stat(logger.LogLocation); !os.IsNotExist(err) { + if err = os.Remove(logger.LogLocation); err != nil { + t.Fatal(err) + } + } defer func() { if err = os.RemoveAll(filepath.Dir(logger.LogLocation)); err != nil { t.Fatal(err) @@ -488,7 +493,6 @@ func TestOperationLogHandler_Restore(t *testing.T) { err = operationLogHandler.RestoreWithTime(time.Date(2024, 8, 11, 20, 20, 0, 0, location), time.Date(2024, 8, 11, 23, 20, 0, 0, location)) assert.Error(t, err) - t.Log(err) value, err := db.Get([]byte("test")) assert.Error(t, err) From 048e496f517b91b23f7cab9671dc2f01d3aa0870 Mon Sep 17 00:00:00 2001 From: halalala222 <1741196223@qq.com> Date: Mon, 12 Aug 2024 15:38:43 +0800 Subject: [PATCH 6/6] fix: while import logger,db will create log dir cause TestColumn_ListColumnFamilies failed --- db/column/column_test.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/db/column/column_test.go b/db/column/column_test.go index a5561ae..149daae 100644 --- a/db/column/column_test.go +++ b/db/column/column_test.go @@ -4,6 +4,7 @@ import ( "github.com/ByteStorage/FlyDB/config" "github.com/ByteStorage/FlyDB/lib/wal" "github.com/stretchr/testify/assert" + "io/ioutil" "os" "testing" ) @@ -75,6 +76,15 @@ func TestColumn_ListColumnFamilies(t *testing.T) { assert.Nil(t, err) assert.NotNil(t, column) + dirs, err := ioutil.ReadDir(option.DbMemoryOptions.Option.DirPath) + assert.Nil(t, err) + dirCount := 0 + for _, dir := range dirs { + if dir.IsDir() { + dirCount++ + } + } + err = column.CreateColumnFamily("test") assert.Nil(t, err) @@ -89,7 +99,7 @@ func TestColumn_ListColumnFamilies(t *testing.T) { list, err := column.ListColumnFamilies() assert.Nil(t, err) - assert.Equal(t, 4, len(list)) + assert.Equal(t, dirCount+3, len(list)) err = column.DropColumnFamily("test") assert.Nil(t, err)