server: add fields to migrate from 3.5 to 3.4

Signed-off-by: Bogdan Kanivets <[email protected]>

etcdutl/etcdutl/migrate_command.go

@@ -87,8 +87,8 @@ func (o *migrateOptions) Config() (*migrateConfig, error) {
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse target version: %v", err)
 	}
-	if c.targetVersion.LessThan(version.V3_5) {
-		return nil, fmt.Errorf(`target version %q not supported. Minimal "3.5"`, storageVersionToString(c.targetVersion))
+	if c.targetVersion.LessThan(version.V3_4) {
+		return nil, fmt.Errorf(`target version %q not supported. Minimal "3.4"`, storageVersionToString(c.targetVersion))
 	}
 
 	dbPath := datadir.ToBackendFileName(o.dataDir)

scripts/test.sh

@@ -559,22 +559,42 @@ function dep_pass {
 
 function release_pass {
   rm -f ./bin/etcd-last-release
+  mkdir -p ./bin
   # to grab latest patch release; bump this up for every minor release
-  UPGRADE_VER=$(git tag -l --sort=-version:refname "v3.5.*" | head -1 | cut -d- -f1)
-  if [ -n "${MANUAL_VER:-}" ]; then
+  UPGRADE_VER_LAST=$(git tag -l --sort=-version:refname "v3.5.*" | head -1 | cut -d- -f1)
+  if [ -n "${MANUAL_VER_LAST:-}" ]; then
     # in case, we need to test against different version
-    UPGRADE_VER=$MANUAL_VER
+    UPGRADE_VER_LAST=$MANUAL_VER_LAST
   fi
-  if [[ -z ${UPGRADE_VER} ]]; then
-    UPGRADE_VER="v3.5.0"
-    log_warning "fallback to" ${UPGRADE_VER}
+  if [[ -z ${UPGRADE_VER_LAST} ]]; then
+    UPGRADE_VER_LAST="v3.5.0"
+    log_warning "fallback to" ${UPGRADE_VER_LAST}
   fi
 
-  local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz"
+  download_etcd_ver_to_tem ${UPGRADE_VER_LAST}
+  mv /tmp/etcd ./bin/etcd-last-release
+
+  UPGRADE_VER_BEFORE_LAST=$(git tag -l --sort=-version:refname "v3.4.*" | head -1 | cut -d- -f1)
+  if [ -n "${MANUAL_VER_BEFORE_LAST:-}" ]; then
+    # in case, we need to test against different version
+    UPGRADE_VER_BEFORE_LAST=$MANUAL_VER_BEFORE_LAST
+  fi
+  if [[ -z ${UPGRADE_VER_BEFORE_LAST} ]]; then
+    UPGRADE_VER_BEFORE_LAST="v3.4.0"
+    log_warning "fallback to" ${UPGRADE_VER_BEFORE_LAST}
+  fi
+
+  download_etcd_ver_to_tem ${UPGRADE_VER_BEFORE_LAST}
+  mv /tmp/etcd ./bin/etcd-before-last-release
+}
+
+function download_etcd_ver_to_tem {
+  local version="$1"
+  local file="etcd-$version-linux-$GOARCH.tar.gz"
   log_callout "Downloading $file"
 
   set +e
-  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file"
+  curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$version/$file" -o "/tmp/$file"
   local result=$?
   set -e
   case $result in
@@ -585,8 +605,6 @@ function release_pass {
   esac
 
   tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1
-  mkdir -p ./bin
-  mv /tmp/etcd ./bin/etcd-last-release
 }
 
 function mod_tidy_for_module {

server/storage/schema/bucket.go

@@ -69,16 +69,16 @@ func (b bucket) IsSafeRangeBucket() bool { return b.safeRangeBucket }
 
 var (
 	// Pre v3.5
-	ScheduledCompactKeyName    = []byte("scheduledCompactRev")
-	FinishedCompactKeyName     = []byte("finishedCompactRev")
-	MetaConsistentIndexKeyName = []byte("consistent_index")
-	AuthEnabledKeyName         = []byte("authEnabled")
-	AuthRevisionKeyName        = []byte("authRevision")
-	// Since v3.5
-	MetaTermKeyName              = []byte("term")
-	MetaConfStateName            = []byte("confState")
+	ScheduledCompactKeyName      = []byte("scheduledCompactRev")
+	FinishedCompactKeyName       = []byte("finishedCompactRev")
+	MetaConsistentIndexKeyName   = []byte("consistent_index")
+	AuthEnabledKeyName           = []byte("authEnabled")
+	AuthRevisionKeyName          = []byte("authRevision")
 	ClusterClusterVersionKeyName = []byte("clusterVersion")
-	ClusterDowngradeKeyName      = []byte("downgrade")
+	// Since v3.5
+	MetaTermKeyName         = []byte("term")
+	MetaConfStateName       = []byte("confState")
+	ClusterDowngradeKeyName = []byte("downgrade")
 	// Since v3.6
 	MetaStorageVersionName = []byte("storageVersion")
 	// Before adding new meta key please update server/etcdserver/version

server/storage/schema/confstate.go

@@ -49,6 +49,10 @@ func UnsafeConfStateFromBackend(lg *zap.Logger, tx backend.ReadTx) *raftpb.ConfS
 			zap.Int("number-of-key", len(keys)),
 		)
 	}
+	if len(vals[0]) == 0 {
+		// confState can be empty. For example after `migrate` from 3.4 to 3.5
+		return nil
+	}
 	var confState raftpb.ConfState
 	if err := json.Unmarshal(vals[0], &confState); err != nil {
 		log.Panic("Cannot unmarshal confState json retrieved from the backend",

server/storage/schema/membership.go

@@ -183,7 +183,7 @@ func mustParseMemberIDFromBytes(lg *zap.Logger, key []byte) types.ID {
 }
 
 // ClusterVersionFromBackend reads cluster version from backend.
-// The field is populated since etcd v3.5.
+// The field is populated since etcd v3.4.
 func (s *membershipBackend) ClusterVersionFromBackend() *semver.Version {
 	ckey := ClusterClusterVersionKeyName
 	tx := s.be.ReadTx()
@@ -220,6 +220,10 @@ func (s *membershipBackend) DowngradeInfoFromBackend() *version.DowngradeInfo {
 			zap.Int("number-of-key", len(keys)),
 		)
 	}
+
+	if len(vals[0]) == 0 {
+		return nil
+	}
 	var d version.DowngradeInfo
 	if err := json.Unmarshal(vals[0], &d); err != nil {
 		s.lg.Panic("failed to unmarshal downgrade information", zap.Error(err))

server/storage/schema/membership_test.go

@@ -0,0 +1,47 @@
+// Copyright 2023 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package schema
+
+import (
+	"github.com/stretchr/testify/assert"
+	"go.etcd.io/etcd/api/v3/version"
+	serverversion "go.etcd.io/etcd/server/v3/etcdserver/version"
+	betesting "go.etcd.io/etcd/server/v3/storage/backend/testing"
+	"go.uber.org/zap/zaptest"
+	"testing"
+)
+
+func TestDowngradeInfoFromBackend(t *testing.T) {
+	lg := zaptest.NewLogger(t)
+	be, _ := betesting.NewDefaultTmpBackend(t)
+	defer betesting.Close(t, be)
+
+	mbe := NewMembershipBackend(lg, be)
+
+	mbe.MustCreateBackendBuckets()
+	mbe.be.ForceCommit()
+	assert.Nil(t, mbe.DowngradeInfoFromBackend())
+
+	mbe.be.BatchTx().UnsafePut(Cluster, ClusterDowngradeKeyName, []byte(""))
+	mbe.be.ForceCommit()
+	assert.Nil(t, mbe.DowngradeInfoFromBackend())
+
+	dinfo := &serverversion.DowngradeInfo{Enabled: true, TargetVersion: version.V3_5.String()}
+	mbe.MustSaveDowngradeToBackend(dinfo)
+	mbe.be.ForceCommit()
+	info := mbe.DowngradeInfoFromBackend()
+
+	assert.Equal(t, dinfo, info)
+}

server/storage/schema/schema.go

@@ -83,7 +83,7 @@ func UnsafeMigrate(lg *zap.Logger, tx backend.BatchTx, w WALVersion, target semv
 // DetectSchemaVersion returns version of storage schema. Returned value depends on etcd version that created the backend. For
 // * v3.6 and newer will return storage version.
 // * v3.5 will return it's version if it includes all storage fields added in v3.5 (might require a snapshot).
-// * v3.4 and older is not supported and will return error.
+// * v3.4 will return it's version if it doesn't include all storage fields added in v3.5.
 func DetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Version, err error) {
 	tx.RLock()
 	defer tx.RUnlock()
@@ -96,12 +96,15 @@ func UnsafeDetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Vers
 	if vp != nil {
 		return *vp, nil
 	}
+
 	confstate := UnsafeConfStateFromBackend(lg, tx)
-	if confstate == nil {
-		return v, fmt.Errorf("missing confstate information")
-	}
 	_, term := UnsafeReadConsistentIndex(tx)
-	if term == 0 {
+	if confstate == nil && term == 0 {
+		// if both confstate and term are missing, assume it's v3.4
+		return version.V3_4, nil
+	} else if confstate == nil {
+		return v, fmt.Errorf("missing confstate information")
+	} else if term == 0 {
 		return v, fmt.Errorf("missing term information")
 	}
 	return version.V3_5, nil
@@ -131,8 +134,16 @@ var (
 		version.V3_6: {
 			addNewField(Meta, MetaStorageVersionName, emptyStorageVersion),
 		},
+		version.V3_5: {
+			// it's safe to set emptyValue
+			// UnsafeReadConsistentIndex, UnsafeConfStateFromBackend, DowngradeInfoFromBackend checks for zero-length values
+			addNewField(Meta, MetaTermKeyName, emptyValue),
+			addNewField(Meta, MetaConfStateName, emptyValue),
+			addNewField(Cluster, ClusterDowngradeKeyName, emptyValue),
+		},
 	}
 	// emptyStorageVersion is used for v3.6 Step for the first time, in all other version StoragetVersion should be set by migrator.
 	// Adding a addNewField for StorageVersion we can reuse logic to remove it when downgrading to v3.5
 	emptyStorageVersion = []byte("")
+	emptyValue          = []byte("")
 )

server/storage/schema/schema_test.go

@@ -105,71 +105,73 @@ func TestMigrate(t *testing.T) {
 		targetVersion semver.Version
 		walEntries    []etcdserverpb.InternalRaftRequest
 
-		expectVersion  *semver.Version
-		expectError    bool
-		expectErrorMsg string
+		expectedStorageVersion *semver.Version
+		expectError            bool
+		expectErrorMsg         string
 	}{
 		// As storage version field was added in v3.6, for v3.5 we will not set it.
 		// For storage to be considered v3.5 it have both confstate and term key set.
 		{
-			name:           `Upgrading v3.5 to v3.6 should be rejected if confstate is not set`,
-			version:        version.V3_5,
-			overrideKeys:   func(tx backend.BatchTx) {},
-			targetVersion:  version.V3_6,
-			expectVersion:  nil,
-			expectError:    true,
-			expectErrorMsg: `cannot detect storage schema version: missing confstate information`,
+			name:    `Upgrading v3.5 to v3.6 should be rejected if confstate is not set`,
+			version: version.V3_5,
+			overrideKeys: func(tx backend.BatchTx) {
+				UnsafeUpdateConsistentIndex(tx, 1, 1)
+			},
+			targetVersion:          version.V3_6,
+			expectedStorageVersion: nil,
+			expectError:            true,
+			expectErrorMsg:         `cannot detect storage schema version: missing confstate information`,
 		},
 		{
 			name:    `Upgrading v3.5 to v3.6 should be rejected if term is not set`,
 			version: version.V3_5,
 			overrideKeys: func(tx backend.BatchTx) {
 				MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{})
 			},
-			targetVersion:  version.V3_6,
-			expectVersion:  nil,
-			expectError:    true,
-			expectErrorMsg: `cannot detect storage schema version: missing term information`,
+			targetVersion:          version.V3_6,
+			expectedStorageVersion: nil,
+			expectError:            true,
+			expectErrorMsg:         `cannot detect storage schema version: missing term information`,
 		},
 		{
-			name:          `Upgrading v3.5 to v3.6 should succeed; all required fields are set`,
-			version:       version.V3_5,
-			targetVersion: version.V3_6,
-			expectVersion: &version.V3_6,
+			name:                   `Upgrading v3.5 to v3.6 should succeed; all required fields are set`,
+			version:                version.V3_5,
+			targetVersion:          version.V3_6,
+			expectedStorageVersion: &version.V3_6,
 		},
 		{
-			name:          `Migrate on same v3.5 version passes and doesn't set storage version'`,
-			version:       version.V3_5,
-			targetVersion: version.V3_5,
-			expectVersion: nil,
+			name:                   `Migrate on same v3.5 version passes and doesn't set storage version'`,
+			version:                version.V3_5,
+			targetVersion:          version.V3_5,
+			expectedStorageVersion: nil,
 		},
 		{
-			name:          `Migrate on same v3.6 version passes`,
-			version:       version.V3_6,
-			targetVersion: version.V3_6,
-			expectVersion: &version.V3_6,
+			name:                   `Migrate on same v3.6 version passes`,
+			version:                version.V3_6,
+			targetVersion:          version.V3_6,
+			expectedStorageVersion: &version.V3_6,
 		},
 		{
-			name:          `Migrate on same v3.7 version passes`,
-			version:       version.V3_7,
-			targetVersion: version.V3_7,
-			expectVersion: &version.V3_7,
+			name:                   `Migrate on same v3.7 version passes`,
+			version:                version.V3_7,
+			targetVersion:          version.V3_7,
+			expectedStorageVersion: &version.V3_7,
 		},
 		{
-			name:           "Upgrading 3.6 to v3.7 is not supported",
-			version:        version.V3_6,
-			targetVersion:  version.V3_7,
-			expectVersion:  &version.V3_6,
-			expectError:    true,
-			expectErrorMsg: `cannot create migration plan: version "3.7.0" is not supported`,
+			name:                   "Upgrading 3.6 to v3.7 is not supported",
+			version:                version.V3_6,
+			targetVersion:          version.V3_7,
+			expectedStorageVersion: &version.V3_6,
+			expectError:            true,
+			expectErrorMsg:         `cannot create migration plan: version "3.7.0" is not supported`,
 		},
 		{
-			name:           "Downgrading v3.7 to v3.6 is not supported",
-			version:        version.V3_7,
-			targetVersion:  version.V3_6,
-			expectVersion:  &version.V3_7,
-			expectError:    true,
-			expectErrorMsg: `cannot create migration plan: version "3.7.0" is not supported`,
+			name:                   "Downgrading v3.7 to v3.6 is not supported",
+			version:                version.V3_7,
+			targetVersion:          version.V3_6,
+			expectedStorageVersion: &version.V3_7,
+			expectError:            true,
+			expectErrorMsg:         `cannot create migration plan: version "3.7.0" is not supported`,
 		},
 		{
 			name:          "Downgrading v3.6 to v3.5 works as there are no v3.6 wal entries",
@@ -178,7 +180,7 @@ func TestMigrate(t *testing.T) {
 			walEntries: []etcdserverpb.InternalRaftRequest{
 				{Range: &etcdserverpb.RangeRequest{Key: []byte("\x00"), RangeEnd: []byte("\xff")}},
 			},
-			expectVersion: nil,
+			expectedStorageVersion: nil,
 		},
 		{
 			name:          "Downgrading v3.6 to v3.5 fails if there are newer WAL entries",
@@ -187,17 +189,23 @@ func TestMigrate(t *testing.T) {
 			walEntries: []etcdserverpb.InternalRaftRequest{
 				{ClusterVersionSet: &membershippb.ClusterVersionSetRequest{Ver: "3.6.0"}},
 			},
-			expectVersion:  &version.V3_6,
-			expectError:    true,
-			expectErrorMsg: "cannot downgrade storage, WAL contains newer entries",
+			expectedStorageVersion: &version.V3_6,
+			expectError:            true,
+			expectErrorMsg:         "cannot downgrade storage, WAL contains newer entries",
 		},
 		{
-			name:           "Downgrading v3.5 to v3.4 is not supported as schema was introduced in v3.6",
-			version:        version.V3_5,
-			targetVersion:  version.V3_4,
-			expectVersion:  nil,
-			expectError:    true,
-			expectErrorMsg: `cannot create migration plan: version "3.5.0" is not supported`,
+			name:          "Downgrading v3.5 to v3.4 is supported",
+			version:       version.V3_5,
+			targetVersion: version.V3_4,
+			// note: 3.4 doesn't have storageVersion, this field was added in 3.6
+			expectedStorageVersion: nil,
+		},
+		{
+			name:          `Upgrading v3.4 to v3.5 should succeed`,
+			version:       version.V3_4,
+			targetVersion: version.V3_5,
+			// note: 3.5 doesn't have storageVersion, this field was added in 3.6
+			expectedStorageVersion: nil,
 		},
 	}
 	for _, tc := range tcs {
@@ -222,7 +230,7 @@ func TestMigrate(t *testing.T) {
 				t.Errorf("Migrate(lg, tx, %q) = %q, expected error message: %q", tc.targetVersion, err, tc.expectErrorMsg)
 			}
 			v := UnsafeReadStorageVersion(b.BatchTx())
-			assert.Equal(t, tc.expectVersion, v)
+			assert.Equal(t, tc.expectedStorageVersion, v)
 		})
 	}
 }
@@ -303,6 +311,7 @@ func setupBackendData(t *testing.T, ver semver.Version, overrideKeys func(tx bac
 	}
 	tx.Lock()
 	UnsafeCreateMetaBucket(tx)
+	tx.UnsafeCreateBucket(Cluster)
 	if overrideKeys != nil {
 		overrideKeys(tx)
 	} else {