Remove unused code

I purposefully left unused struct methods, and also unused code inside `library/go` untouched.

---

Pull Request resolved: #183

Co-authored-by: tserakhau <[email protected]>
commit_hash:fb8a197f5b14768ec7070cf1ea2a5ee938097e48

.mapping.json

@@ -1824,7 +1824,6 @@
   "pkg/providers/yt/lfstaging/close_gaps_test.go":"transfer_manager/go/pkg/providers/yt/lfstaging/close_gaps_test.go",
   "pkg/providers/yt/lfstaging/intermediate_writer.go":"transfer_manager/go/pkg/providers/yt/lfstaging/intermediate_writer.go",
   "pkg/providers/yt/lfstaging/intermediate_writer_test.go":"transfer_manager/go/pkg/providers/yt/lfstaging/intermediate_writer_test.go",
-  "pkg/providers/yt/lfstaging/lock.go":"transfer_manager/go/pkg/providers/yt/lfstaging/lock.go",
   "pkg/providers/yt/lfstaging/logbroker_metadata.go":"transfer_manager/go/pkg/providers/yt/lfstaging/logbroker_metadata.go",
   "pkg/providers/yt/lfstaging/logbroker_metadata_test.go":"transfer_manager/go/pkg/providers/yt/lfstaging/logbroker_metadata_test.go",
   "pkg/providers/yt/lfstaging/rows.go":"transfer_manager/go/pkg/providers/yt/lfstaging/rows.go",

internal/logger/common.go

@@ -59,26 +59,3 @@ func copySlice(src []byte) []byte {
 	copy(dst, src)
 	return dst
 }
-
-func copyBytes(source []byte) []byte {
-	dup := make([]byte, len(source))
-	copy(dup, source)
-	return dup
-}
-
-func newConsoleLogger() log.Logger {
-	levels := getEnvLogLevels()
-	if os.Getenv("CONSOLE_LOG_LEVEL") != "" {
-		levels = parseLevel(os.Getenv("CONSOLE_LOG_LEVEL"))
-	}
-
-	levelEnabler := levelEnablerFactory(levels.Zap)
-
-	encoderConfig := zap.CLIConfig(levels.Log).EncoderConfig
-	encoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
-
-	encoder := zapcore.NewConsoleEncoder(encoderConfig)
-	writeSyncer := zapcore.AddSync(os.Stdout)
-
-	return newLogger(zapcore.NewCore(encoder, writeSyncer, levelEnabler))
-}

pkg/abstract/model/endpoint_rotator_config_test.go

@@ -35,11 +35,6 @@ var monthList = []time.Month{
 	time.June, time.July, time.August, time.September, time.October, time.November, time.December,
 }
 
-var (
-	monthDayCount     = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
-	monthDayCountLeap = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
-)
-
 // scenario tests
 func scenarioTesting(t *testing.T) {
 	// this tests based on real user scenarios
@@ -90,6 +85,7 @@ func scenarioTestingDTSUPPORT693(t *testing.T) {
 		timestamp = timestamp.Add(time.Hour)
 		require.Equal(t, rotator.KeepPartCount, len(rotationTables), "Check that there is always window of six tables")
 	}
+
 }
 
 // tests
@@ -179,18 +175,18 @@ func getMonthPartitionedTestHeavy(t *testing.T) {
 			require.Equal(t, monthList[i-(i%partSize)], rc.getMonthPartitioned(month))
 		}
 	}
+
 }
 
 func offsetDateTest(t *testing.T) {
 	t.Setenv("TZ", "Europe/Moscow") // this test is timezone aware
 	t.Run("Hours", offsetDateTestHours)
 	t.Run("Days", offsetDateTestDays)
-	// t.Run("MonthHeavy", offsetDateTestMonthHeavy) // TODO(@kry127) temporary switched off
+	//t.Run("MonthHeavy", offsetDateTestMonthHeavy) // TODO(@kry127) temporary switched off
 	t.Run("NilReceiver", offsetDateTestNilReceiver)
 }
 
 func offsetDateTestHours(t *testing.T) {
-	t.Parallel()
 	rcHours := RotatorConfig{KeepPartCount: 0, PartType: RotatorPartHour, PartSize: 1, TimeColumn: ""}
 	rcHoursTimestamp := time.Now()
 
@@ -209,7 +205,6 @@ func offsetDateTestHours(t *testing.T) {
 }
 
 func offsetDateTestDays(t *testing.T) {
-	t.Parallel()
 	rcDays := RotatorConfig{KeepPartCount: 0, PartType: RotatorPartDay, PartSize: 1, TimeColumn: ""}
 	rcDaysTimestamp := time.Now()
 
@@ -249,6 +244,11 @@ func isLeap(year int) bool {
 }
 
 func countDaysForYearAcc(year, month, offset int, acc int64) int64 {
+	var (
+		monthDayCount     = []int{31, 28, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
+		monthDayCountLeap = []int{31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31}
+	)
+
 	switch {
 	case offset > 0:
 		dayDiff := 0
@@ -288,11 +288,9 @@ func countDaysForYear(year, month, offset int) int64 {
 	return countDaysForYearAcc(year, month, offset, 0)
 }
 
-func offsetDateTestMonthHeavy(t *testing.T) {
-	t.Parallel()
+func TestOffsetDateTestMonthHeavy(t *testing.T) {
 	checkYear := func(t *testing.T, year, partSize int) {
 		t.Helper()
-		t.Parallel()
 		rcMonths := RotatorConfig{KeepPartCount: 0, PartType: RotatorPartMonth, PartSize: partSize, TimeColumn: ""}
 		nowTimestamp := time.Now()
 		for offset := 1; offset < 15; offset++ {

pkg/csv/error.go

@@ -4,7 +4,6 @@ import "github.com/doublecloud/transfer/library/go/core/xerrors"
 
 var (
 	errInvalidDelimiter     = xerrors.NewSentinel("csv: invalid delimiter")
-	errInvalidEscape        = xerrors.NewSentinel("csv: escape char used outside of quoted text")
 	errDoubleQuotesDisabled = xerrors.NewSentinel("csv: found double quotes while double quote feature disabled")
 	errQuotingDisabled      = xerrors.NewSentinel("csv: found quote char while feature disabled")
 )

pkg/debezium/typeutil/helpers.go

@@ -21,10 +21,11 @@ import (
 // it's fixed in higher version of debezium (in 1.1 this bug is present, in 1.8 absent)
 // so, actual function - changeItemsBitsToDebeziumHonest
 
-func changeItemsBitsToDebeziumWA(bits string) string {
-	bufSize := imitateDebeziumBufSize(len(bits))
-	return changeItemsBitsStringToDebezium(bits, bufSize)
-}
+/*
+ func changeItemsBitsToDebeziumWA(bits string) string {
+	 bufSize := imitateDebeziumBufSize(len(bits))
+	 return changeItemsBitsStringToDebezium(bits, bufSize)
+ }
 
 func imitateDebeziumBufSize(bitsCount int) int {
 	if bitsCount < 16 {
@@ -37,6 +38,7 @@ func imitateDebeziumBufSize(bitsCount int) int {
 		return int(math.Ceil(float64(bitsCount) / 8)) // honest count
 	}
 }
+*/
 
 //---------------------------------------------------------------------------------------------------------------------
 
@@ -116,10 +118,12 @@ func GetTimeDivider(originalTypeWithoutProvider string) (int, error) {
 	}
 }
 
-var reTimeWithoutTZ = regexp.MustCompile(`^time\((\d)\) without time zone`)
-var reTimestampWithoutTZ = regexp.MustCompile(`^timestamp\((\d)\) without time zone`)
-var reMysqlTime = regexp.MustCompile(`^mysql:timestamp\((\d)\)`)
-var reMysqlDatetime = regexp.MustCompile(`^mysql:datetime\((\d)\)`)
+var (
+	reTimeWithoutTZ      = regexp.MustCompile(`^time\((\d)\) without time zone`)
+	reTimestampWithoutTZ = regexp.MustCompile(`^timestamp\((\d)\) without time zone`)
+	reMysqlTime          = regexp.MustCompile(`^mysql:timestamp\((\d)\)`)
+	reMysqlDatetime      = regexp.MustCompile(`^mysql:datetime\((\d)\)`)
+)
 
 func GetTimePrecision(colTypeStr string) int {
 	precision := -1
@@ -447,10 +451,12 @@ func DecimalToDebeziumPrimitives(decimal string, connectorParameters map[string]
 	}
 }
 
-var pgTimestampLayout0 = "2006-01-02T15:04:05Z"
-var pgTimestampLayout1 = "2006-01-02 15:04:05Z"
-var pgTimestampLayout2 = "2006-01-02T15:04:05-07:00"
-var pgTimestampLayout3 = "2006-01-02 15:04:05-07"
+var (
+	pgTimestampLayout0 = "2006-01-02T15:04:05Z"
+	pgTimestampLayout1 = "2006-01-02 15:04:05Z"
+	pgTimestampLayout2 = "2006-01-02T15:04:05-07:00"
+	pgTimestampLayout3 = "2006-01-02 15:04:05-07"
+)
 
 func ParsePgDateTimeWithTimezone(in string) (time.Time, error) {
 	var result time.Time
@@ -664,10 +670,12 @@ func ParsePgDateTimeWithTimezone2(l, r string) (time.Time, time.Time, error) {
 	return lTime, rTime, nil
 }
 
-var regexHour = *regexp.MustCompile(`(\d+):(\d+):(\d+)`)
-var regexYear = *regexp.MustCompile(`(\d+) years?`)
-var regexMonth = *regexp.MustCompile(`(\d+) (?:mon|months?)`)
-var regexDay = *regexp.MustCompile(`(\d+) days?`)
+var (
+	regexHour  = *regexp.MustCompile(`(\d+):(\d+):(\d+)`)
+	regexYear  = *regexp.MustCompile(`(\d+) years?`)
+	regexMonth = *regexp.MustCompile(`(\d+) (?:mon|months?)`)
+	regexDay   = *regexp.MustCompile(`(\d+) days?`)
+)
 
 func ExtractPostgresIntervalArray(interval string) ([]string, error) {
 	result := make([]string, 7)
@@ -706,13 +714,15 @@ func ExtractPostgresIntervalArray(interval string) ([]string, error) {
 	return result, nil
 }
 
-var timeWithoutTZ0 = "15:04:05"
-var timeWithoutTZ1 = "15:04:05.0"
-var timeWithoutTZ2 = "15:04:05.00"
-var timeWithoutTZ3 = "15:04:05.000"
-var timeWithoutTZ4 = "15:04:05.0000"
-var timeWithoutTZ5 = "15:04:05.00000"
-var timeWithoutTZ6 = "15:04:05.000000"
+var (
+	timeWithoutTZ0 = "15:04:05"
+	timeWithoutTZ1 = "15:04:05.0"
+	timeWithoutTZ2 = "15:04:05.00"
+	timeWithoutTZ3 = "15:04:05.000"
+	timeWithoutTZ4 = "15:04:05.0000"
+	timeWithoutTZ5 = "15:04:05.00000"
+	timeWithoutTZ6 = "15:04:05.000000"
+)
 
 func ParseTimeWithoutTZ(timeStr string) (time.Time, error) {
 	var layout string
@@ -742,13 +752,15 @@ func ParseTimeWithoutTZ(timeStr string) (time.Time, error) {
 	return timeVal, nil
 }
 
-var timestampWithoutTZ0 = "2006-01-02T15:04:05Z"
-var timestampWithoutTZ1 = "2006-01-02T15:04:05.0Z"
-var timestampWithoutTZ2 = "2006-01-02T15:04:05.00Z"
-var timestampWithoutTZ3 = "2006-01-02T15:04:05.000Z"
-var timestampWithoutTZ4 = "2006-01-02T15:04:05.0000Z"
-var timestampWithoutTZ5 = "2006-01-02T15:04:05.00000Z"
-var timestampWithoutTZ6 = "2006-01-02T15:04:05.000000Z"
+var (
+	timestampWithoutTZ0 = "2006-01-02T15:04:05Z"
+	timestampWithoutTZ1 = "2006-01-02T15:04:05.0Z"
+	timestampWithoutTZ2 = "2006-01-02T15:04:05.00Z"
+	timestampWithoutTZ3 = "2006-01-02T15:04:05.000Z"
+	timestampWithoutTZ4 = "2006-01-02T15:04:05.0000Z"
+	timestampWithoutTZ5 = "2006-01-02T15:04:05.00000Z"
+	timestampWithoutTZ6 = "2006-01-02T15:04:05.000000Z"
+)
 
 func ParseTimestamp(timeStr string) (time.Time, error) {
 	var layout string
@@ -832,12 +844,14 @@ func BufToChangeItemsBits(in []byte) string {
 	return result
 }
 
-var yearMS = int64(31557600000000)
-var monthMS = int64(2629800000000)
-var dayMS = int64(3600 * 24 * 1000 * 1000)
-var hourMS = int64(3600 * 1000 * 1000)
-var minuteMS = int64(60 * 1000 * 1000)
-var secondMS = int64(1000 * 1000)
+var (
+	yearMS   = int64(31557600000000)
+	monthMS  = int64(2629800000000)
+	dayMS    = int64(3600 * 24 * 1000 * 1000)
+	hourMS   = int64(3600 * 1000 * 1000)
+	minuteMS = int64(60 * 1000 * 1000)
+	secondMS = int64(1000 * 1000)
+)
 
 func EmitPostgresInterval(val int64) string {
 	y := val / yearMS
@@ -1106,8 +1120,10 @@ func OriginalTypeWithoutProvider(originalType string) string {
 	return originalType[index+1:]
 }
 
-var pgTimeWithoutTimeZoneParam = *regexp.MustCompile(`pg:time\((\d)\) without time zone`)
-var pgNumeric = *regexp.MustCompile(`pg:numeric\(\d+,\d+\)`)
+var (
+	pgTimeWithoutTimeZoneParam = *regexp.MustCompile(`pg:time\((\d)\) without time zone`)
+	pgNumeric                  = *regexp.MustCompile(`pg:numeric\(\d+,\d+\)`)
+)
 
 func PgTimeWithoutTimeZonePrecision(originalType string) int {
 	if originalType == "pg:time without time zone" {

pkg/parsers/tests/generic_parser_test.go

@@ -140,24 +140,6 @@ func TestParser_TableSplitterSpecChar(t *testing.T) {
 	require.Equal(t, test6TableName[:len(test6TableExpectedPrefix)], test6TableExpectedPrefix)
 }
 
-func changeItemFactoryForMetrica(tableName string, appVersionName string, receiveTimestamp int64, apiKey int32, deviceID interface{}) *abstract.ChangeItem {
-	table1Schema := abstract.NewTableSchema(abstract.TableColumns{
-		abstract.MakeTypedColSchema("AppVersionName", string(schema.TypeString), true),
-		abstract.MakeTypedColSchema("ReceiveTimestamp", string(schema.TypeInt64), false),
-		abstract.MakeTypedColSchema("APIKey", string(schema.TypeInt32), false),
-		abstract.MakeTypedColSchema("DeviceID", string(schema.TypeAny), false),
-	})
-	item1 := abstract.ChangeItem{
-		Kind:         "Insert",
-		Schema:       "db",
-		Table:        tableName,
-		ColumnNames:  []string{"AppVersionName", "ReceiveTimestamp", "APIKey", "DeviceID"},
-		ColumnValues: []interface{}{appVersionName, receiveTimestamp, apiKey, deviceID},
-		TableSchema:  table1Schema,
-	}
-	return &item1
-}
-
 func testTableSplitterOnChangeItem(
 	t *testing.T,
 	originalTableName string,
@@ -224,7 +206,6 @@ func TestGenericParser_DoSensitive(t *testing.T) {
 				fieldsWithSecretErasure++
 			}
 		}
-
 	}
 	logger.Log.Debug("Example of secret erasure", log.Any("res", res))
 	require.Greater(t, fieldsWithSecretErasure, 0, "there should be at least one erasured secret")
@@ -245,7 +226,6 @@ func TestGenericParser_DoSensitiveDisabled(t *testing.T) {
 				fieldsWithSecretErasure++
 			}
 		}
-
 	}
 	logger.Log.Debug("Example of keeping secrets unerasured to increase performance", log.Any("res", res))
 	require.Equal(t, 0, fieldsWithSecretErasure, "secrets should not be processed in order to increase performance")
@@ -440,9 +420,9 @@ func TestGenericParser_Parse_vs_Do(t *testing.T) {
 				return
 			}
 			require.NoError(t, err)
-			//genericParserImpl := GetGenericParserImpl(parser)
+			// genericParserImpl := GetGenericParserImpl(parser)
 			res := parser.Do(samples.Data[k], *new(abstract.Partition))
-			//batch := genericParserImpl.Parse(samples.Data[k], *new(abstract.Partition))
+			// batch := genericParserImpl.Parse(samples.Data[k], *new(abstract.Partition))
 			var changes []abstract.ChangeItem
 			//for batch.Next() {
 			//	ev, err := batch.Event()
@@ -458,7 +438,7 @@ func TestGenericParser_Parse_vs_Do(t *testing.T) {
 			abstract.Dump(res)
 			t.Logf("new parser: %v len", len(changes))
 			abstract.Dump(changes)
-			//require.Equal(t, len(res), len(changes))
+			// require.Equal(t, len(res), len(changes))
 		})
 	}
 }

pkg/parsers/utils.go

@@ -19,10 +19,8 @@ import (
 )
 
 type schema struct {
-	Path     ypath.Path
-	Fields   []abstract.ColSchema `json:"fields"`
-	revision int
-	dead     chan bool
+	Path   ypath.Path
+	Fields []abstract.ColSchema `json:"fields"`
 }
 
 func adjustType(s string) string {

pkg/predicate/token.go

@@ -80,11 +80,3 @@ func (tok Token) Precedence() int {
 
 // isOperator returns true for operator tokens.
 func (tok Token) isOperator() bool { return tok > operatorBegin && tok < operatorEnd }
-
-// tokstr returns a literal if provided, otherwise returns the token string.
-func tokstr(tok Token, lit string) string {
-	if lit != "" {
-		return lit
-	}
-	return tok.String()
-}

pkg/providers/clickhouse/schema/engine.go

@@ -165,32 +165,6 @@ func ParseMergeTreeFamilyEngine(sql string) (*MergeTreeFamilyEngine, string, err
 	return GetEngine(engineStr)
 }
 
-type substr struct {
-	startIdx int
-	endIdx   int
-}
-
-func grepEngineFull(sql string) (substr, error) {
-	startIdx := strings.Index(sql, "ENGINE = ")
-	if startIdx == -1 {
-		return substr{}, xerrors.Errorf("Cannot find engine definition in '%v'", sql)
-	}
-	startIdx += len("ENGINE = ")
-
-	endIdx := -1
-	if endIdx = TryFindNextStatement(sql, startIdx); endIdx == -1 {
-		if endIdx = strings.Index(sql[startIdx:], ")"); endIdx == -1 {
-			if endIdx = strings.Index(sql[startIdx:], " "); endIdx == -1 {
-				return substr{}, xerrors.Errorf("Cannot find engine definition in '%v'", sql)
-			}
-		} else {
-			endIdx += 1 // need include closing parenthesis
-		}
-		endIdx += startIdx
-	}
-	return substr{startIdx: startIdx, endIdx: endIdx}, nil
-}
-
 func TryFindNextStatement(sql string, from int) int {
 	possibleStatements := []string{
 		" ORDER BY",