Merge pull request #26 from stellar/patch/fix-ttl-bug

Add deduplication for ttl data

Author: amishas157

internal/transform/ttl.go

@@ -30,6 +30,10 @@ func GetTTLDataDetails(changes []ingest.Change, lhe xdr.LedgerHeaderHistoryEntry
 		ttlDataOutputs = append(ttlDataOutputs, TransformTTLData)
 
 	}
+
+	// It is possible to have multiple changes to the same ttl entry in a single ledger
+	// example from testnet data: CDO7SMNK3H2ZTRWSLJOPMGFLHDN5SKNSWJI6CNB2TBXCXAKFC6DPTTZY, 83c830c6d200adbceda8e72d8204017f781b57e1ec8acf03674388a902732779, 901
+	ttlDataOutputs = utils.RemoveDuplicatesByFields(ttlDataOutputs, []string{"KeyHash", "LedgerSequence"})
 	return ttlDataOutputs, nil
 }
 

internal/transform/ttl_test.go

@@ -107,3 +107,137 @@ func makeTtlTestOutput() []contract.TtlOutput {
 		},
 	}
 }
+
+// TestGetTTLDetailsWithDuplicates tests the deduplication behavior when multiple changes
+// to the same TTL entry occur within a single ledger, preventing regression of issue #25
+func TestGetTTLDetailsWithDuplicates(t *testing.T) {
+	var hash xdr.Hash
+	expectedKeyHash := "0000000000000000000000000000000000000000000000000000000000000000"
+	ledgerSeq := xdr.Uint32(10)
+
+	// Simulate multiple changes to the same TTL entry within a single ledger
+	// Each change updates the LiveUntilLedgerSeq value
+	// In practice, this can happen when the same entry is bumped multiple times
+	// within the same ledger by different operations
+
+	// First change: initial update to LiveUntilLedgerSeq = 100
+	preTtlLedgerEntry1 := xdr.LedgerEntry{
+		LastModifiedLedgerSeq: ledgerSeq - 1,
+		Data: xdr.LedgerEntryData{
+			Type: xdr.LedgerEntryTypeTtl,
+			Ttl: &xdr.TtlEntry{
+				KeyHash:            hash,
+				LiveUntilLedgerSeq: 0,
+			},
+		},
+	}
+
+	postTtlLedgerEntry1 := xdr.LedgerEntry{
+		LastModifiedLedgerSeq: ledgerSeq,
+		Data: xdr.LedgerEntryData{
+			Type: xdr.LedgerEntryTypeTtl,
+			Ttl: &xdr.TtlEntry{
+				KeyHash:            hash,
+				LiveUntilLedgerSeq: 100,
+			},
+		},
+	}
+
+	// Second change: another update to the same KeyHash, LiveUntilLedgerSeq = 200
+	preTtlLedgerEntry2 := xdr.LedgerEntry{
+		LastModifiedLedgerSeq: ledgerSeq,
+		Data: xdr.LedgerEntryData{
+			Type: xdr.LedgerEntryTypeTtl,
+			Ttl: &xdr.TtlEntry{
+				KeyHash:            hash,
+				LiveUntilLedgerSeq: 100,
+			},
+		},
+	}
+
+	postTtlLedgerEntry2 := xdr.LedgerEntry{
+		LastModifiedLedgerSeq: ledgerSeq,
+		Data: xdr.LedgerEntryData{
+			Type: xdr.LedgerEntryTypeTtl,
+			Ttl: &xdr.TtlEntry{
+				KeyHash:            hash,
+				LiveUntilLedgerSeq: 200,
+			},
+		},
+	}
+
+	// Third change: final update to the same KeyHash, LiveUntilLedgerSeq = 300
+	preTtlLedgerEntry3 := xdr.LedgerEntry{
+		LastModifiedLedgerSeq: ledgerSeq,
+		Data: xdr.LedgerEntryData{
+			Type: xdr.LedgerEntryTypeTtl,
+			Ttl: &xdr.TtlEntry{
+				KeyHash:            hash,
+				LiveUntilLedgerSeq: 200,
+			},
+		},
+	}
+
+	postTtlLedgerEntry3 := xdr.LedgerEntry{
+		LastModifiedLedgerSeq: ledgerSeq,
+		Data: xdr.LedgerEntryData{
+			Type: xdr.LedgerEntryTypeTtl,
+			Ttl: &xdr.TtlEntry{
+				KeyHash:            hash,
+				LiveUntilLedgerSeq: 300,
+			},
+		},
+	}
+
+	// Create multiple changes to the same TTL entry within a single ledger
+	changes := []ingest.Change{
+		{
+			ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
+			Type:       xdr.LedgerEntryTypeTtl,
+			Pre:        &preTtlLedgerEntry1,
+			Post:       &postTtlLedgerEntry1,
+		},
+		{
+			ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
+			Type:       xdr.LedgerEntryTypeTtl,
+			Pre:        &preTtlLedgerEntry2,
+			Post:       &postTtlLedgerEntry2,
+		},
+		{
+			ChangeType: xdr.LedgerEntryChangeTypeLedgerEntryUpdated,
+			Type:       xdr.LedgerEntryTypeTtl,
+			Pre:        &preTtlLedgerEntry3,
+			Post:       &postTtlLedgerEntry3,
+		},
+	}
+
+	header := xdr.LedgerHeaderHistoryEntry{
+		Header: xdr.LedgerHeader{
+			ScpValue: xdr.StellarValue{
+				CloseTime: 1000,
+			},
+			LedgerSeq: ledgerSeq,
+		},
+	}
+
+	actualOutput, actualError := GetTTLDataDetails(changes, header)
+
+	// Should not error
+	assert.NoError(t, actualError)
+
+	// Should only have 1 entry after deduplication (the latest one)
+	assert.Equal(t, 1, len(actualOutput))
+
+	// The retained entry should be the latest one with LiveUntilLedgerSeq = 300
+	expectedOutput := contract.TtlOutput{
+		KeyHash:            expectedKeyHash,
+		LiveUntilLedgerSeq: 300,
+		LastModifiedLedger: uint32(ledgerSeq),
+		LedgerEntryChange:  1, // Updated entry
+		Deleted:            false,
+		LedgerSequence:     uint32(ledgerSeq),
+		ClosedAt:           time.Date(1970, time.January, 1, 0, 16, 40, 0, time.UTC),
+	}
+
+	assert.Equal(t, expectedOutput, actualOutput[0])
+}