From 853bebf386fd53a02cadb5bba06d335ac0e15044 Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Thu, 29 Aug 2024 13:15:38 +0200 Subject: [PATCH 1/9] Basic cleanup and unit test harness --- inline_verifier.go | 4 +-- inline_verifier_test.go | 55 +++++++++++++++++++++++++++++++++++++++++ iterative_verifier.go | 2 +- 3 files changed, 58 insertions(+), 3 deletions(-) create mode 100644 inline_verifier_test.go diff --git a/inline_verifier.go b/inline_verifier.go index 80d56bab..78bd692b 100644 --- a/inline_verifier.go +++ b/inline_verifier.go @@ -593,7 +593,7 @@ func (v *InlineVerifier) compareHashes(source, target map[uint64][]byte) map[uin return mismatchSet } -func (v *InlineVerifier) compareDecompressedData(source, target map[uint64]map[string][]byte) map[uint64]struct{} { +func compareDecompressedData(source, target map[uint64]map[string][]byte) map[uint64]struct{} { mismatchSet := map[uint64]struct{}{} for paginationKey, targetDecompressedColumns := range target { @@ -633,7 +633,7 @@ func (v *InlineVerifier) compareDecompressedData(source, target map[uint64]map[s func (v *InlineVerifier) compareHashesAndData(sourceHashes, targetHashes map[uint64][]byte, sourceData, targetData map[uint64]map[string][]byte) []InlineVerifierMismatches { mismatches := v.compareHashes(sourceHashes, targetHashes) - compressedMismatch := v.compareDecompressedData(sourceData, targetData) + compressedMismatch := compareDecompressedData(sourceData, targetData) for paginationKey, _ := range compressedMismatch { mismatches[paginationKey] = InlineVerifierMismatches{ Pk: paginationKey, diff --git a/inline_verifier_test.go b/inline_verifier_test.go new file mode 100644 index 00000000..de7cd191 --- /dev/null +++ b/inline_verifier_test.go @@ -0,0 +1,55 @@ +package ghostferry + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestCompareDecompressedDataNoDifference(t *testing.T) { + source := map[uint64]map[string][]byte{ + 31: {"name": []byte("Leszek")}, + } + target := map[uint64]map[string][]byte{ + 31: {"name": []byte("Leszek")}, + } + + result := compareDecompressedData(source, target) + + assert.Equal(t, map[uint64]struct{}{}, result) +} + +func TestCompareDecompressedDataContentDifference(t *testing.T) { + source := map[uint64]map[string][]byte{ + 1: {"name": []byte("Leszek")}, + } + target := map[uint64]map[string][]byte{ + 1: {"name": []byte("Steve")}, + } + + result := compareDecompressedData(source, target) + + assert.Equal(t, map[uint64]struct{}{1: {}}, result) +} + +func TestCompareDecompressedDataMissingTarget(t *testing.T) { + source := map[uint64]map[string][]byte{ + 1: {"name": []byte("Leszek")}, + } + target := map[uint64]map[string][]byte{} + + result := compareDecompressedData(source, target) + + assert.Equal(t, map[uint64]struct{}{1: {}}, result) +} + +func TestCompareDecompressedDataMissingSource(t *testing.T) { + source := map[uint64]map[string][]byte{} + target := map[uint64]map[string][]byte{ + 3: {"name": []byte("Leszek")}, + } + + result := compareDecompressedData(source, target) + + assert.Equal(t, map[uint64]struct{}{3: {}}, result) +} diff --git a/iterative_verifier.go b/iterative_verifier.go index 94aafd58..114bf35e 100644 --- a/iterative_verifier.go +++ b/iterative_verifier.go @@ -337,7 +337,7 @@ func (v *IterativeVerifier) reverifyUntilStoreIsSmallEnough(maxIterations int) e before := v.reverifyStore.RowCount start := time.Now() - _, err := v.verifyStore("reverification_before_cutover", []MetricTag{{"iteration", string(iteration)}}) + _, err := v.verifyStore("reverification_before_cutover", []MetricTag{{"iteration", strconv.FormatInt(int64(iteration), 10)}}) if err != nil { return err } From 7d4ce23fc20a62e032609d65f6de7a81f102f884 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20K=C3=BClzer?= Date: Wed, 28 Aug 2024 14:07:44 -0400 Subject: [PATCH 2/9] added mismatches to InlineVerifierMismatches --- inline_verifier.go | 64 +++++++++++++++++++++++++++++++++-------- inline_verifier_test.go | 8 +++--- 2 files changed, 56 insertions(+), 16 deletions(-) diff --git a/inline_verifier.go b/inline_verifier.go index 78bd692b..ee16cab6 100644 --- a/inline_verifier.go +++ b/inline_verifier.go @@ -236,6 +236,7 @@ type InlineVerifierMismatches struct { Pk uint64 SourceChecksum string TargetChecksum string + Mismatch mismatch } type InlineVerifier struct { @@ -593,20 +594,51 @@ func (v *InlineVerifier) compareHashes(source, target map[uint64][]byte) map[uin return mismatchSet } -func compareDecompressedData(source, target map[uint64]map[string][]byte) map[uint64]struct{} { - mismatchSet := map[uint64]struct{}{} +type mismatchType string + +const ( + MismatchColumnMissingOnSource mismatchType = "column missing on source" + MismatchColumnMissingOnTarget mismatchType = "column missing on target" + MismatchRowMissingOnSource mismatchType = "row missing on source" + MismatchRowMissingOnTarget mismatchType = "row missing on target" + MismatchContentDifference mismatchType = "content difference" +) + +type mismatch struct { + paginationKey uint64 + column string + mismatchType mismatchType +} + +func compareDecompressedData(source, target map[uint64]map[string][]byte) map[uint64]mismatch { + mismatchSet := map[uint64]mismatch{} for paginationKey, targetDecompressedColumns := range target { sourceDecompressedColumns, exists := source[paginationKey] if !exists { - mismatchSet[paginationKey] = struct{}{} + // row missing on source + mismatchSet[paginationKey] = mismatch{ + paginationKey: paginationKey, + mismatchType: MismatchRowMissingOnSource, + } continue } for colName, targetData := range targetDecompressedColumns { sourceData, exists := sourceDecompressedColumns[colName] - if !exists || !bytes.Equal(sourceData, targetData) { - mismatchSet[paginationKey] = struct{}{} + if !exists { + mismatchSet[paginationKey] = mismatch{ + paginationKey: paginationKey, + column: colName, + mismatchType: MismatchColumnMissingOnSource, + } + break // no need to compare other columns + } else if !bytes.Equal(sourceData, targetData) { + mismatchSet[paginationKey] = mismatch{ + paginationKey: paginationKey, + column: colName, + mismatchType: MismatchContentDifference, + } break // no need to compare other columns } } @@ -615,15 +647,22 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui for paginationKey, sourceDecompressedColumns := range source { targetDecompressedColumns, exists := target[paginationKey] if !exists { - mismatchSet[paginationKey] = struct{}{} + // row missing on target + mismatchSet[paginationKey] = mismatch{ + paginationKey: paginationKey, + mismatchType: MismatchRowMissingOnTarget, + } continue } - for colName, sourceData := range sourceDecompressedColumns { - targetData, exists := targetDecompressedColumns[colName] - if !exists || !bytes.Equal(sourceData, targetData) { - mismatchSet[paginationKey] = struct{}{} - break + for colName := range sourceDecompressedColumns { + _, exists := targetDecompressedColumns[colName] + if !exists { + mismatchSet[paginationKey] = mismatch{ + paginationKey: paginationKey, + column: colName, + mismatchType: MismatchColumnMissingOnTarget, + } } } } @@ -634,11 +673,12 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui func (v *InlineVerifier) compareHashesAndData(sourceHashes, targetHashes map[uint64][]byte, sourceData, targetData map[uint64]map[string][]byte) []InlineVerifierMismatches { mismatches := v.compareHashes(sourceHashes, targetHashes) compressedMismatch := compareDecompressedData(sourceData, targetData) - for paginationKey, _ := range compressedMismatch { + for paginationKey, mismatch := range compressedMismatch { mismatches[paginationKey] = InlineVerifierMismatches{ Pk: paginationKey, SourceChecksum: "compressed-data-mismatch", // TODO: compute the hash of the compressed data and put it here TargetChecksum: "compressed-data-mismatch", + Mismatch: mismatch, } } diff --git a/inline_verifier_test.go b/inline_verifier_test.go index de7cd191..e9877450 100644 --- a/inline_verifier_test.go +++ b/inline_verifier_test.go @@ -16,7 +16,7 @@ func TestCompareDecompressedDataNoDifference(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]struct{}{}, result) + assert.Equal(t, map[uint64]mismatch{}, result) } func TestCompareDecompressedDataContentDifference(t *testing.T) { @@ -29,7 +29,7 @@ func TestCompareDecompressedDataContentDifference(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]struct{}{1: {}}, result) + assert.Equal(t, map[uint64]mismatch{1: {paginationKey: 1, mismatchType: MismatchContentDifference, column: "name"}}, result) } func TestCompareDecompressedDataMissingTarget(t *testing.T) { @@ -40,7 +40,7 @@ func TestCompareDecompressedDataMissingTarget(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]struct{}{1: {}}, result) + assert.Equal(t, map[uint64]mismatch{1: {paginationKey: 1, mismatchType: MismatchRowMissingOnTarget}}, result) } func TestCompareDecompressedDataMissingSource(t *testing.T) { @@ -51,5 +51,5 @@ func TestCompareDecompressedDataMissingSource(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]struct{}{3: {}}, result) + assert.Equal(t, map[uint64]mismatch{3: {paginationKey: 3, mismatchType: MismatchRowMissingOnSource}}, result) } From 6fd817cbe3f613423a1dd0ffdadbd0433e4362c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jakob=20K=C3=BClzer?= Date: Wed, 28 Aug 2024 14:10:26 -0400 Subject: [PATCH 3/9] added todo --- inline_verifier.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/inline_verifier.go b/inline_verifier.go index ee16cab6..a05641bf 100644 --- a/inline_verifier.go +++ b/inline_verifier.go @@ -448,6 +448,8 @@ func (v *InlineVerifier) VerifyDuringCutover() (VerificationResult, error) { }, nil } + // TODO make this into a func: + // func FormatMismatches(mismatches map[string]map[string][]InlineVerifierMismatches)) string // Build error message for display var messageBuf bytes.Buffer messageBuf.WriteString("cutover verification failed for: ") @@ -465,6 +467,10 @@ func (v *InlineVerifier) VerifyDuringCutover() (VerificationResult, error) { messageBuf.WriteString(mismatch.SourceChecksum) messageBuf.WriteString(", target: ") messageBuf.WriteString(mismatch.TargetChecksum) + // TODO add the mismatch details somehow + // ([paginationKeys: (source: "..", target: "..")]) + // to + // ([paginationKeys: (source: "..", target: "..", type: "missing on source", column: "")]) messageBuf.WriteString(") ") } messageBuf.WriteString("] ") @@ -476,7 +482,7 @@ func (v *InlineVerifier) VerifyDuringCutover() (VerificationResult, error) { return VerificationResult{ DataCorrect: false, - Message: messageBuf.String(), + Message: message, IncorrectTables: incorrectTables, }, nil } From 2dd8c90c6429e88a8e03b81eebf73f94b3ed944a Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Tue, 3 Sep 2024 14:31:48 +0200 Subject: [PATCH 4/9] formatMessage for the verify result --- inline_verifier.go | 75 ++++++++++++++++++++++----------------- inline_verifier_test.go | 78 +++++++++++++++++++++++++++++++++++++++-- 2 files changed, 117 insertions(+), 36 deletions(-) diff --git a/inline_verifier.go b/inline_verifier.go index a05641bf..2f47f293 100644 --- a/inline_verifier.go +++ b/inline_verifier.go @@ -5,6 +5,7 @@ import ( "context" "errors" "fmt" + "sort" "strconv" "strings" "sync" @@ -433,51 +434,65 @@ func (v *InlineVerifier) VerifyBeforeCutover() error { return nil } -func (v *InlineVerifier) VerifyDuringCutover() (VerificationResult, error) { - v.verifyDuringCutoverStarted.Set(true) - - mismatchFound, mismatches, err := v.verifyAllEventsInStore() - if err != nil { - v.logger.WithError(err).Error("failed to VerifyDuringCutover") - return VerificationResult{}, err - } - - if !mismatchFound { - return VerificationResult{ - DataCorrect: true, - }, nil - } - - // TODO make this into a func: - // func FormatMismatches(mismatches map[string]map[string][]InlineVerifierMismatches)) string +func formatMismatches(mismatches map[string]map[string][]InlineVerifierMismatches) (string, []string) { // Build error message for display var messageBuf bytes.Buffer messageBuf.WriteString("cutover verification failed for: ") incorrectTables := make([]string, 0) + for schemaName, _ := range mismatches { - for tableName, mismatches := range mismatches[schemaName] { - tableName = fmt.Sprintf("%s.%s", schemaName, tableName) - incorrectTables = append(incorrectTables, tableName) + sortedTables := make([]string, 0, len(mismatches[schemaName])) + for tableName, _ := range mismatches[schemaName] { + sortedTables = append(sortedTables, tableName) + } + sort.Strings(sortedTables) + + for _, tableName := range sortedTables { + tableNameWithSchema := fmt.Sprintf("%s.%s", schemaName, tableName) + incorrectTables = append(incorrectTables, tableNameWithSchema) - messageBuf.WriteString(tableName) + messageBuf.WriteString(tableNameWithSchema) messageBuf.WriteString(" [paginationKeys: ") - for _, mismatch := range mismatches { + for _, mismatch := range mismatches[schemaName][tableName] { messageBuf.WriteString(strconv.FormatUint(mismatch.Pk, 10)) messageBuf.WriteString(" (source: ") messageBuf.WriteString(mismatch.SourceChecksum) messageBuf.WriteString(", target: ") messageBuf.WriteString(mismatch.TargetChecksum) - // TODO add the mismatch details somehow - // ([paginationKeys: (source: "..", target: "..")]) - // to - // ([paginationKeys: (source: "..", target: "..", type: "missing on source", column: "")]) + messageBuf.WriteString(", type: ") + messageBuf.WriteString(string(mismatch.Mismatch.mismatchType)) + if mismatch.Mismatch.column != "" { + messageBuf.WriteString(", column: ") + messageBuf.WriteString(mismatch.Mismatch.column) + } + messageBuf.WriteString(") ") } messageBuf.WriteString("] ") } } - message := messageBuf.String() + return messageBuf.String(), incorrectTables +} + +func (v *InlineVerifier) VerifyDuringCutover() (VerificationResult, error) { + v.verifyDuringCutoverStarted.Set(true) + + mismatchFound, mismatches, err := v.verifyAllEventsInStore() + + if err != nil { + v.logger.WithError(err).Error("failed to VerifyDuringCutover") + return VerificationResult{}, err + } + + if !mismatchFound { + return VerificationResult{ + DataCorrect: true, + }, nil + } + + message, incorrectTables := formatMismatches(mismatches) + v.logger.WithField("incorrect_tables", incorrectTables).Error(message) return VerificationResult{ @@ -611,7 +626,6 @@ const ( ) type mismatch struct { - paginationKey uint64 column string mismatchType mismatchType } @@ -624,7 +638,6 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui if !exists { // row missing on source mismatchSet[paginationKey] = mismatch{ - paginationKey: paginationKey, mismatchType: MismatchRowMissingOnSource, } continue @@ -634,14 +647,12 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui sourceData, exists := sourceDecompressedColumns[colName] if !exists { mismatchSet[paginationKey] = mismatch{ - paginationKey: paginationKey, column: colName, mismatchType: MismatchColumnMissingOnSource, } break // no need to compare other columns } else if !bytes.Equal(sourceData, targetData) { mismatchSet[paginationKey] = mismatch{ - paginationKey: paginationKey, column: colName, mismatchType: MismatchContentDifference, } @@ -655,7 +666,6 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui if !exists { // row missing on target mismatchSet[paginationKey] = mismatch{ - paginationKey: paginationKey, mismatchType: MismatchRowMissingOnTarget, } continue @@ -665,7 +675,6 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui _, exists := targetDecompressedColumns[colName] if !exists { mismatchSet[paginationKey] = mismatch{ - paginationKey: paginationKey, column: colName, mismatchType: MismatchColumnMissingOnTarget, } diff --git a/inline_verifier_test.go b/inline_verifier_test.go index e9877450..44c2b930 100644 --- a/inline_verifier_test.go +++ b/inline_verifier_test.go @@ -29,7 +29,7 @@ func TestCompareDecompressedDataContentDifference(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]mismatch{1: {paginationKey: 1, mismatchType: MismatchContentDifference, column: "name"}}, result) + assert.Equal(t, map[uint64]mismatch{1: {mismatchType: MismatchContentDifference, column: "name"}}, result) } func TestCompareDecompressedDataMissingTarget(t *testing.T) { @@ -40,7 +40,7 @@ func TestCompareDecompressedDataMissingTarget(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]mismatch{1: {paginationKey: 1, mismatchType: MismatchRowMissingOnTarget}}, result) + assert.Equal(t, map[uint64]mismatch{1: {mismatchType: MismatchRowMissingOnTarget}}, result) } func TestCompareDecompressedDataMissingSource(t *testing.T) { @@ -51,5 +51,77 @@ func TestCompareDecompressedDataMissingSource(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]mismatch{3: {paginationKey: 3, mismatchType: MismatchRowMissingOnSource}}, result) + assert.Equal(t, map[uint64]mismatch{3: {mismatchType: MismatchRowMissingOnSource}}, result) +} + +func TestFormatMismatch(t *testing.T) { + mismatches := map[string]map[string][]InlineVerifierMismatches{ + "default": { + "users": { + InlineVerifierMismatches{ + Pk: 1, + SourceChecksum: "", + TargetChecksum: "bar", + Mismatch: mismatch{ + mismatchType: MismatchRowMissingOnSource, + }, + }, + }, + }, + } + message, tables := formatMismatches(mismatches) + + assert.Equal(t, string("cutover verification failed for: default.users [paginationKeys: 1 (source: , target: bar, type: row missing on source) ] "), message) + assert.Equal(t, []string{string("default.users")}, tables) +} + +func TestFormatMismatches(t *testing.T) { + mismatches := map[string]map[string][]InlineVerifierMismatches{ + "default": { + "users": { + InlineVerifierMismatches{ + Pk: 1, + SourceChecksum: "", + TargetChecksum: "bar", + Mismatch: mismatch{ + mismatchType: MismatchRowMissingOnSource, + }, + }, + InlineVerifierMismatches{ + Pk: 5, + SourceChecksum: "baz", + TargetChecksum: "", + Mismatch: mismatch{ + mismatchType: MismatchRowMissingOnTarget, + }, + }, + }, + "posts": { + InlineVerifierMismatches{ + Pk: 9, + SourceChecksum: "boo", + TargetChecksum: "aaa", + Mismatch: mismatch{ + mismatchType: MismatchContentDifference, + column: string("title"), + }, + }, + }, + "attachments": { + InlineVerifierMismatches{ + Pk: 7, + SourceChecksum: "boo", + TargetChecksum: "aaa", + Mismatch: mismatch{ + mismatchType: MismatchContentDifference, + column: string("name"), + }, + }, + }, + }, + } + message, tables := formatMismatches(mismatches) + + assert.Equal(t, string("cutover verification failed for: default.attachments [paginationKeys: 7 (source: boo, target: aaa, type: content difference, column: name) ] default.posts [paginationKeys: 9 (source: boo, target: aaa, type: content difference, column: title) ] default.users [paginationKeys: 1 (source: , target: bar, type: row missing on source) 5 (source: baz, target: , type: row missing on target) ] "), message) + assert.Equal(t, []string{string("default.attachments"), string("default.posts"), string("default.users")}, tables) } From 304534187ca39251443ba4151e62198a91cad8cc Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Tue, 3 Sep 2024 15:13:15 +0200 Subject: [PATCH 5/9] Run standard go test as well --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 3c88a1ef..e9480578 100644 --- a/Makefile +++ b/Makefile @@ -64,6 +64,7 @@ test-go: fi ulimit -n 1024 && ./bin/gotestsum --format short-verbose ./test/go ./copydb/test ./sharding/test -count 1 -p 1 -failfast + go test -v test-ruby: bundle install From 53c5c66e82ef8b75dd74c3ab7f641a15d64da7fd Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Tue, 10 Sep 2024 12:48:08 +0200 Subject: [PATCH 6/9] Squash types, make sure mismatchType is always there --- inline_verifier.go | 117 +++++++++++++---------- inline_verifier_test.go | 50 ++++------ test/integration/inline_verifier_test.rb | 49 ++++++++-- 3 files changed, 127 insertions(+), 89 deletions(-) diff --git a/inline_verifier.go b/inline_verifier.go index 2f47f293..be2c8422 100644 --- a/inline_verifier.go +++ b/inline_verifier.go @@ -3,6 +3,8 @@ package ghostferry import ( "bytes" "context" + "crypto/md5" + "encoding/hex" "errors" "fmt" "sort" @@ -233,11 +235,23 @@ func (s *BinlogVerifyStore) Serialize() BinlogVerifySerializedStore { return s.store.Copy() } +type mismatchType string + +const ( + MismatchColumnMissingOnSource mismatchType = "column missing on source" + MismatchColumnMissingOnTarget mismatchType = "column missing on target" + MismatchRowMissingOnSource mismatchType = "row missing on source" + MismatchRowMissingOnTarget mismatchType = "row missing on target" + MismatchContentDifference mismatchType = "content difference" + MismatchChecksumDifference mismatchType = "rows checksum difference" +) + type InlineVerifierMismatches struct { Pk uint64 SourceChecksum string TargetChecksum string - Mismatch mismatch + MismatchColumn string + MismatchType mismatchType } type InlineVerifier struct { @@ -452,18 +466,23 @@ func formatMismatches(mismatches map[string]map[string][]InlineVerifierMismatche incorrectTables = append(incorrectTables, tableNameWithSchema) messageBuf.WriteString(tableNameWithSchema) - messageBuf.WriteString(" [paginationKeys: ") + messageBuf.WriteString(" [PKs: ") for _, mismatch := range mismatches[schemaName][tableName] { messageBuf.WriteString(strconv.FormatUint(mismatch.Pk, 10)) - messageBuf.WriteString(" (source: ") - messageBuf.WriteString(mismatch.SourceChecksum) - messageBuf.WriteString(", target: ") - messageBuf.WriteString(mismatch.TargetChecksum) - messageBuf.WriteString(", type: ") - messageBuf.WriteString(string(mismatch.Mismatch.mismatchType)) - if mismatch.Mismatch.column != "" { + messageBuf.WriteString(" (type: ") + messageBuf.WriteString(string(mismatch.MismatchType)) + if mismatch.SourceChecksum != "" { + messageBuf.WriteString(", source: ") + messageBuf.WriteString(mismatch.SourceChecksum) + } + if mismatch.TargetChecksum != "" { + messageBuf.WriteString(", target: ") + messageBuf.WriteString(mismatch.TargetChecksum) + } + + if mismatch.MismatchColumn != "" { messageBuf.WriteString(", column: ") - messageBuf.WriteString(mismatch.Mismatch.column) + messageBuf.WriteString(mismatch.MismatchColumn) } messageBuf.WriteString(") ") @@ -592,22 +611,27 @@ func (v *InlineVerifier) compareHashes(source, target map[uint64][]byte) map[uin for paginationKey, targetHash := range target { sourceHash, exists := source[paginationKey] - if !bytes.Equal(sourceHash, targetHash) || !exists { + if !exists { mismatchSet[paginationKey] = InlineVerifierMismatches{ Pk: paginationKey, + MismatchType: MismatchRowMissingOnSource, + } + } else if !bytes.Equal(sourceHash, targetHash) { + mismatchSet[paginationKey] = InlineVerifierMismatches{ + Pk: paginationKey, + MismatchType: MismatchChecksumDifference, SourceChecksum: string(sourceHash), TargetChecksum: string(targetHash), } } } - for paginationKey, sourceHash := range source { - targetHash, exists := target[paginationKey] - if !bytes.Equal(sourceHash, targetHash) || !exists { + for paginationKey, _ := range source { + _, exists := target[paginationKey] + if !exists { mismatchSet[paginationKey] = InlineVerifierMismatches{ Pk: paginationKey, - SourceChecksum: string(sourceHash), - TargetChecksum: string(targetHash), + MismatchType: MismatchRowMissingOnTarget, } } } @@ -615,30 +639,17 @@ func (v *InlineVerifier) compareHashes(source, target map[uint64][]byte) map[uin return mismatchSet } -type mismatchType string -const ( - MismatchColumnMissingOnSource mismatchType = "column missing on source" - MismatchColumnMissingOnTarget mismatchType = "column missing on target" - MismatchRowMissingOnSource mismatchType = "row missing on source" - MismatchRowMissingOnTarget mismatchType = "row missing on target" - MismatchContentDifference mismatchType = "content difference" -) - -type mismatch struct { - column string - mismatchType mismatchType -} - -func compareDecompressedData(source, target map[uint64]map[string][]byte) map[uint64]mismatch { - mismatchSet := map[uint64]mismatch{} +func compareDecompressedData(source, target map[uint64]map[string][]byte) map[uint64]InlineVerifierMismatches { + mismatchSet := map[uint64]InlineVerifierMismatches{} for paginationKey, targetDecompressedColumns := range target { sourceDecompressedColumns, exists := source[paginationKey] if !exists { // row missing on source - mismatchSet[paginationKey] = mismatch{ - mismatchType: MismatchRowMissingOnSource, + mismatchSet[paginationKey] = InlineVerifierMismatches{ + Pk: paginationKey, + MismatchType: MismatchRowMissingOnSource, } continue } @@ -646,15 +657,22 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui for colName, targetData := range targetDecompressedColumns { sourceData, exists := sourceDecompressedColumns[colName] if !exists { - mismatchSet[paginationKey] = mismatch{ - column: colName, - mismatchType: MismatchColumnMissingOnSource, + mismatchSet[paginationKey] = InlineVerifierMismatches{ + Pk: paginationKey, + MismatchType: MismatchColumnMissingOnSource, + MismatchColumn: colName, } break // no need to compare other columns } else if !bytes.Equal(sourceData, targetData) { - mismatchSet[paginationKey] = mismatch{ - column: colName, - mismatchType: MismatchContentDifference, + sourceChecksum := md5.Sum(sourceData) + targetChecksum := md5.Sum(targetData) + + mismatchSet[paginationKey] = InlineVerifierMismatches{ + Pk: paginationKey, + MismatchType: MismatchContentDifference, + MismatchColumn: colName, + SourceChecksum: hex.EncodeToString(sourceChecksum[:]), + TargetChecksum: hex.EncodeToString(targetChecksum[:]), } break // no need to compare other columns } @@ -665,8 +683,9 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui targetDecompressedColumns, exists := target[paginationKey] if !exists { // row missing on target - mismatchSet[paginationKey] = mismatch{ - mismatchType: MismatchRowMissingOnTarget, + mismatchSet[paginationKey] = InlineVerifierMismatches{ + Pk: paginationKey, + MismatchType: MismatchRowMissingOnTarget, } continue } @@ -674,9 +693,10 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui for colName := range sourceDecompressedColumns { _, exists := targetDecompressedColumns[colName] if !exists { - mismatchSet[paginationKey] = mismatch{ - column: colName, - mismatchType: MismatchColumnMissingOnTarget, + mismatchSet[paginationKey] = InlineVerifierMismatches{ + Pk: paginationKey, + MismatchColumn: colName, + MismatchType: MismatchColumnMissingOnTarget, } } } @@ -689,12 +709,7 @@ func (v *InlineVerifier) compareHashesAndData(sourceHashes, targetHashes map[uin mismatches := v.compareHashes(sourceHashes, targetHashes) compressedMismatch := compareDecompressedData(sourceData, targetData) for paginationKey, mismatch := range compressedMismatch { - mismatches[paginationKey] = InlineVerifierMismatches{ - Pk: paginationKey, - SourceChecksum: "compressed-data-mismatch", // TODO: compute the hash of the compressed data and put it here - TargetChecksum: "compressed-data-mismatch", - Mismatch: mismatch, - } + mismatches[paginationKey] = mismatch } mismatchList := make([]InlineVerifierMismatches, 0, len(mismatches)) diff --git a/inline_verifier_test.go b/inline_verifier_test.go index 44c2b930..61c922f2 100644 --- a/inline_verifier_test.go +++ b/inline_verifier_test.go @@ -16,7 +16,7 @@ func TestCompareDecompressedDataNoDifference(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]mismatch{}, result) + assert.Equal(t, map[uint64]InlineVerifierMismatches{}, result) } func TestCompareDecompressedDataContentDifference(t *testing.T) { @@ -29,7 +29,15 @@ func TestCompareDecompressedDataContentDifference(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]mismatch{1: {mismatchType: MismatchContentDifference, column: "name"}}, result) + assert.Equal(t, map[uint64]InlineVerifierMismatches{ + 1: { + Pk: 1, + MismatchType: MismatchContentDifference, + MismatchColumn: "name", + SourceChecksum: "e356a972989f87a1531252cfa2152797", + TargetChecksum: "81b8a1b77068d06e1c8190825253066f", + }, + }, result) } func TestCompareDecompressedDataMissingTarget(t *testing.T) { @@ -40,7 +48,7 @@ func TestCompareDecompressedDataMissingTarget(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]mismatch{1: {mismatchType: MismatchRowMissingOnTarget}}, result) + assert.Equal(t, map[uint64]InlineVerifierMismatches{1: {Pk: 1, MismatchType: MismatchRowMissingOnTarget}}, result) } func TestCompareDecompressedDataMissingSource(t *testing.T) { @@ -51,7 +59,7 @@ func TestCompareDecompressedDataMissingSource(t *testing.T) { result := compareDecompressedData(source, target) - assert.Equal(t, map[uint64]mismatch{3: {mismatchType: MismatchRowMissingOnSource}}, result) + assert.Equal(t, map[uint64]InlineVerifierMismatches{3: {Pk: 3, MismatchType: MismatchRowMissingOnSource}}, result) } func TestFormatMismatch(t *testing.T) { @@ -60,18 +68,14 @@ func TestFormatMismatch(t *testing.T) { "users": { InlineVerifierMismatches{ Pk: 1, - SourceChecksum: "", - TargetChecksum: "bar", - Mismatch: mismatch{ - mismatchType: MismatchRowMissingOnSource, - }, + MismatchType: MismatchRowMissingOnSource, }, }, }, } message, tables := formatMismatches(mismatches) - assert.Equal(t, string("cutover verification failed for: default.users [paginationKeys: 1 (source: , target: bar, type: row missing on source) ] "), message) + assert.Equal(t, string("cutover verification failed for: default.users [PKs: 1 (type: row missing on source) ] "), message) assert.Equal(t, []string{string("default.users")}, tables) } @@ -81,47 +85,35 @@ func TestFormatMismatches(t *testing.T) { "users": { InlineVerifierMismatches{ Pk: 1, - SourceChecksum: "", - TargetChecksum: "bar", - Mismatch: mismatch{ - mismatchType: MismatchRowMissingOnSource, - }, + MismatchType: MismatchRowMissingOnSource, }, InlineVerifierMismatches{ Pk: 5, - SourceChecksum: "baz", - TargetChecksum: "", - Mismatch: mismatch{ - mismatchType: MismatchRowMissingOnTarget, - }, + MismatchType: MismatchRowMissingOnTarget, }, }, "posts": { InlineVerifierMismatches{ Pk: 9, + MismatchType: MismatchContentDifference, + MismatchColumn: string("title"), SourceChecksum: "boo", TargetChecksum: "aaa", - Mismatch: mismatch{ - mismatchType: MismatchContentDifference, - column: string("title"), - }, }, }, "attachments": { InlineVerifierMismatches{ Pk: 7, + MismatchType: MismatchContentDifference, + MismatchColumn: string("name"), SourceChecksum: "boo", TargetChecksum: "aaa", - Mismatch: mismatch{ - mismatchType: MismatchContentDifference, - column: string("name"), - }, }, }, }, } message, tables := formatMismatches(mismatches) - assert.Equal(t, string("cutover verification failed for: default.attachments [paginationKeys: 7 (source: boo, target: aaa, type: content difference, column: name) ] default.posts [paginationKeys: 9 (source: boo, target: aaa, type: content difference, column: title) ] default.users [paginationKeys: 1 (source: , target: bar, type: row missing on source) 5 (source: baz, target: , type: row missing on target) ] "), message) + assert.Equal(t, string("cutover verification failed for: default.attachments [PKs: 7 (type: content difference, source: boo, target: aaa, column: name) ] default.posts [PKs: 9 (type: content difference, source: boo, target: aaa, column: title) ] default.users [PKs: 1 (type: row missing on source) 5 (type: row missing on target) ] "), message) assert.Equal(t, []string{string("default.attachments"), string("default.posts"), string("default.users")}, tables) } diff --git a/test/integration/inline_verifier_test.rb b/test/integration/inline_verifier_test.rb index 26c3a34d..89cc1029 100644 --- a/test/integration/inline_verifier_test.rb +++ b/test/integration/inline_verifier_test.rb @@ -40,7 +40,10 @@ def test_corrupted_insert_is_detected_inline_with_batch_writer assert verification_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: #{corrupting_id}") + + expected_message = "cutover verification failed for: gftest.test_table_1 "\ + "[PKs: #{corrupting_id} (type: rows checksum difference, source: " + assert ghostferry.error_lines.last["msg"].start_with?(expected_message) end def test_different_compressed_data_is_detected_inline_with_batch_writer @@ -68,7 +71,11 @@ def test_different_compressed_data_is_detected_inline_with_batch_writer assert verification_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: 1") + + expected_message = "cutover verification failed for: gftest.test_table_1 "\ + "[PKs: 1 (type: content difference, source: 389101948d1694a3bbfb904f57ae845c, target: 4594bb26f2f93c5c60328df6c86a0846, column: data) ] " + + assert_equal expected_message, ghostferry.error_lines.last["msg"] end def test_same_decompressed_data_different_compressed_test_passes_inline_verification @@ -163,7 +170,11 @@ def test_catches_binlog_streamer_corruption ghostferry.run assert verification_ran - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: #{corrupting_id}") + + expected_message = "cutover verification failed for: gftest.test_table_1 "\ + "[PKs: #{corrupting_id} (type: rows checksum difference, source: ced197ee28c2e73cc737242eb0e8c49c, target: ff030f09c559a197ed440b0eee7950a0) ] " + + assert_equal expected_message, ghostferry.error_lines.last["msg"] end def test_target_corruption_is_ignored_if_skip_target_verification @@ -399,7 +410,11 @@ def test_catches_binlog_streamer_corruption_with_composite_pk ghostferry.run assert verification_ran assert incorrect_tables_found, "verification did not catch corrupted table" - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: #{DEFAULT_DB}.#{DEFAULT_TABLE} [paginationKeys: #{corrupting_id}") + + expected_message = "cutover verification failed for: #{DEFAULT_DB}.#{DEFAULT_TABLE} "\ + "[PKs: #{corrupting_id} (type: rows checksum difference, source: 0cc788986133d5289aba8cd87705d106, target: f4c00525c4daf1388254f1b1024ed35d) ] " + + assert_equal expected_message, ghostferry.error_lines.last["msg"] end def test_positive_negative_zero @@ -430,7 +445,10 @@ def test_positive_negative_zero assert verification_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: #{DEFAULT_DB}.#{DEFAULT_TABLE} [paginationKeys: 1") + + expected_message = "cutover verification failed for: #{DEFAULT_DB}.#{DEFAULT_TABLE} "\ + "[PKs: 1 (type: rows checksum difference, source: 2888f4944da0fba0d5a5c7a7de2346f3, target: 2fa7e7e5e76005ffd8bfa5082da9f2f9) ] " + assert_equal expected_message, ghostferry.error_lines.last["msg"] # Now we run the real test case. target_db.query("UPDATE #{DEFAULT_FULL_TABLE_NAME} SET data = -0.0 WHERE id = 1") @@ -484,7 +502,10 @@ def test_null_vs_empty_string assert verification_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: 1") + + expected_message = "cutover verification failed for: gftest.test_table_1 [PKs: 1 (type: " + + assert ghostferry.error_lines.last["msg"].start_with?(expected_message) end def test_null_vs_null_string @@ -507,7 +528,11 @@ def test_null_vs_null_string assert verification_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: 1") + + expected_message = "cutover verification failed for: gftest.test_table_1 " \ + "[PKs: 1 (type: rows checksum difference, source: 7dfce9db8fc0f2475d2ff8ac3a5382e9, target: dc4cca2441c365c72466c75076782022) ] " + + assert_equal expected_message, ghostferry.error_lines.last["msg"] end def test_null_in_different_order @@ -533,7 +558,11 @@ def test_null_in_different_order assert verification_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: 1") + + expected_message = "cutover verification failed for: gftest.test_table_1 "\ + "[PKs: 1 (type: rows checksum difference, source: 8e8e0931b9b2e5cb422a76d63160bbf3, target: 503b2de936a8da9e8d67b0d4594117d9) ] " + + assert_equal expected_message, ghostferry.error_lines.last["msg"] end ################### @@ -605,7 +634,9 @@ def run_collation_test(data, source_charset, target_charset, identical:) assert verify_during_cutover_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: 1") + + expected_message = "cutover verification failed for: gftest.test_table_1 [PKs: 1 (" + assert ghostferry.error_lines.last["msg"].start_with?(expected_message) end end From 535a3208410b9e2ad4825ae9539a6654ca1051b0 Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Tue, 10 Sep 2024 18:57:11 +0200 Subject: [PATCH 7/9] Use better naming for mismatch types --- inline_verifier.go | 27 ++++++++++++------------ inline_verifier_test.go | 20 +++++++++--------- test/integration/inline_verifier_test.rb | 2 +- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/inline_verifier.go b/inline_verifier.go index be2c8422..552c88e6 100644 --- a/inline_verifier.go +++ b/inline_verifier.go @@ -242,8 +242,8 @@ const ( MismatchColumnMissingOnTarget mismatchType = "column missing on target" MismatchRowMissingOnSource mismatchType = "row missing on source" MismatchRowMissingOnTarget mismatchType = "row missing on target" - MismatchContentDifference mismatchType = "content difference" - MismatchChecksumDifference mismatchType = "rows checksum difference" + MismatchColumnValueDifference mismatchType = "column value difference" + MismatchRowChecksumDifference mismatchType = "rows checksum difference" ) type InlineVerifierMismatches struct { @@ -491,7 +491,7 @@ func formatMismatches(mismatches map[string]map[string][]InlineVerifierMismatche } } - return messageBuf.String(), incorrectTables + return messageBuf.String(), incorrectTables } func (v *InlineVerifier) VerifyDuringCutover() (VerificationResult, error) { @@ -613,13 +613,13 @@ func (v *InlineVerifier) compareHashes(source, target map[uint64][]byte) map[uin sourceHash, exists := source[paginationKey] if !exists { mismatchSet[paginationKey] = InlineVerifierMismatches{ - Pk: paginationKey, - MismatchType: MismatchRowMissingOnSource, + Pk: paginationKey, + MismatchType: MismatchRowMissingOnSource, } } else if !bytes.Equal(sourceHash, targetHash) { mismatchSet[paginationKey] = InlineVerifierMismatches{ Pk: paginationKey, - MismatchType: MismatchChecksumDifference, + MismatchType: MismatchRowChecksumDifference, SourceChecksum: string(sourceHash), TargetChecksum: string(targetHash), } @@ -630,8 +630,8 @@ func (v *InlineVerifier) compareHashes(source, target map[uint64][]byte) map[uin _, exists := target[paginationKey] if !exists { mismatchSet[paginationKey] = InlineVerifierMismatches{ - Pk: paginationKey, - MismatchType: MismatchRowMissingOnTarget, + Pk: paginationKey, + MismatchType: MismatchRowMissingOnTarget, } } } @@ -639,7 +639,6 @@ func (v *InlineVerifier) compareHashes(source, target map[uint64][]byte) map[uin return mismatchSet } - func compareDecompressedData(source, target map[uint64]map[string][]byte) map[uint64]InlineVerifierMismatches { mismatchSet := map[uint64]InlineVerifierMismatches{} @@ -648,8 +647,8 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui if !exists { // row missing on source mismatchSet[paginationKey] = InlineVerifierMismatches{ - Pk: paginationKey, - MismatchType: MismatchRowMissingOnSource, + Pk: paginationKey, + MismatchType: MismatchRowMissingOnSource, } continue } @@ -669,7 +668,7 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui mismatchSet[paginationKey] = InlineVerifierMismatches{ Pk: paginationKey, - MismatchType: MismatchContentDifference, + MismatchType: MismatchColumnValueDifference, MismatchColumn: colName, SourceChecksum: hex.EncodeToString(sourceChecksum[:]), TargetChecksum: hex.EncodeToString(targetChecksum[:]), @@ -684,8 +683,8 @@ func compareDecompressedData(source, target map[uint64]map[string][]byte) map[ui if !exists { // row missing on target mismatchSet[paginationKey] = InlineVerifierMismatches{ - Pk: paginationKey, - MismatchType: MismatchRowMissingOnTarget, + Pk: paginationKey, + MismatchType: MismatchRowMissingOnTarget, } continue } diff --git a/inline_verifier_test.go b/inline_verifier_test.go index 61c922f2..1db37362 100644 --- a/inline_verifier_test.go +++ b/inline_verifier_test.go @@ -31,8 +31,8 @@ func TestCompareDecompressedDataContentDifference(t *testing.T) { assert.Equal(t, map[uint64]InlineVerifierMismatches{ 1: { - Pk: 1, - MismatchType: MismatchContentDifference, + Pk: 1, + MismatchType: MismatchColumnValueDifference, MismatchColumn: "name", SourceChecksum: "e356a972989f87a1531252cfa2152797", TargetChecksum: "81b8a1b77068d06e1c8190825253066f", @@ -67,7 +67,7 @@ func TestFormatMismatch(t *testing.T) { "default": { "users": { InlineVerifierMismatches{ - Pk: 1, + Pk: 1, MismatchType: MismatchRowMissingOnSource, }, }, @@ -84,18 +84,18 @@ func TestFormatMismatches(t *testing.T) { "default": { "users": { InlineVerifierMismatches{ - Pk: 1, + Pk: 1, MismatchType: MismatchRowMissingOnSource, }, InlineVerifierMismatches{ - Pk: 5, + Pk: 5, MismatchType: MismatchRowMissingOnTarget, }, }, "posts": { InlineVerifierMismatches{ - Pk: 9, - MismatchType: MismatchContentDifference, + Pk: 9, + MismatchType: MismatchColumnValueDifference, MismatchColumn: string("title"), SourceChecksum: "boo", TargetChecksum: "aaa", @@ -103,8 +103,8 @@ func TestFormatMismatches(t *testing.T) { }, "attachments": { InlineVerifierMismatches{ - Pk: 7, - MismatchType: MismatchContentDifference, + Pk: 7, + MismatchType: MismatchColumnValueDifference, MismatchColumn: string("name"), SourceChecksum: "boo", TargetChecksum: "aaa", @@ -114,6 +114,6 @@ func TestFormatMismatches(t *testing.T) { } message, tables := formatMismatches(mismatches) - assert.Equal(t, string("cutover verification failed for: default.attachments [PKs: 7 (type: content difference, source: boo, target: aaa, column: name) ] default.posts [PKs: 9 (type: content difference, source: boo, target: aaa, column: title) ] default.users [PKs: 1 (type: row missing on source) 5 (type: row missing on target) ] "), message) + assert.Equal(t, string("cutover verification failed for: default.attachments [PKs: 7 (type: column value difference, source: boo, target: aaa, column: name) ] default.posts [PKs: 9 (type: column value difference, source: boo, target: aaa, column: title) ] default.users [PKs: 1 (type: row missing on source) 5 (type: row missing on target) ] "), message) assert.Equal(t, []string{string("default.attachments"), string("default.posts"), string("default.users")}, tables) } diff --git a/test/integration/inline_verifier_test.rb b/test/integration/inline_verifier_test.rb index 89cc1029..82ce7aef 100644 --- a/test/integration/inline_verifier_test.rb +++ b/test/integration/inline_verifier_test.rb @@ -73,7 +73,7 @@ def test_different_compressed_data_is_detected_inline_with_batch_writer assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables expected_message = "cutover verification failed for: gftest.test_table_1 "\ - "[PKs: 1 (type: content difference, source: 389101948d1694a3bbfb904f57ae845c, target: 4594bb26f2f93c5c60328df6c86a0846, column: data) ] " + "[PKs: 1 (type: column value difference, source: 389101948d1694a3bbfb904f57ae845c, target: 4594bb26f2f93c5c60328df6c86a0846, column: data) ] " assert_equal expected_message, ghostferry.error_lines.last["msg"] end From e6eb592b3375bdab7917e9e6513bbf829e994f9c Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Wed, 11 Sep 2024 13:23:04 +0200 Subject: [PATCH 8/9] Extend tests to have better error messages --- test/integration/interrupt_resume_test.rb | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/test/integration/interrupt_resume_test.rb b/test/integration/interrupt_resume_test.rb index 0da17264..695d0995 100644 --- a/test/integration/interrupt_resume_test.rb +++ b/test/integration/interrupt_resume_test.rb @@ -136,10 +136,11 @@ def test_interrupt_resume_inline_verifier_with_datawriter end dumped_state = ghostferry.run_expecting_interrupt + extra_message = "dumped state was: #{dumped_state.inspect}, expecting something under 'BinlogVerifyStore" assert_basic_fields_exist_in_dumped_state(dumped_state) - refute_nil dumped_state["BinlogVerifyStore"] - refute_nil dumped_state["BinlogVerifyStore"]["gftest"] - refute_nil dumped_state["BinlogVerifyStore"]["gftest"]["test_table_1"] + refute_nil dumped_state["BinlogVerifyStore"], "#{extra_message}'" + refute_nil dumped_state["BinlogVerifyStore"]["gftest"], "#{extra_message}.gftest'" + refute_nil dumped_state["BinlogVerifyStore"]["gftest"]["test_table_1"], "#{extra_message}.gftest.test_table_1'" # Resume Ghostferry with dumped state ghostferry = new_ghostferry(MINIMAL_GHOSTFERRY, config: { verifier_type: "Inline" }) @@ -297,8 +298,11 @@ def test_interrupt_resume_inline_verifier_will_verify_additional_rows_changed_on assert_equal 1, incorrect_tables.length assert_equal "gftest.test_table_1", incorrect_tables.first - error_line = ghostferry.error_lines.last - assert error_line["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: #{chosen_id}") + error_message = ghostferry.error_lines.last["msg"] + predicate = "cutover verification failed for: gftest.test_table_1 [paginationKeys: #{chosen_id}" + expectation = error_message.start_with?(predicate) + + assert expectation, "error message: #{error_message.inspect}, didn't start with #{predicate.inspect}" end # originally taken from @kolbitsch-lastline in https://github.com/Shopify/ghostferry/pull/160 From c8e3c3fd6d08c1192398b7ccb99cd0e556029cbe Mon Sep 17 00:00:00 2001 From: Leszek Zalewski Date: Wed, 11 Sep 2024 14:06:24 +0200 Subject: [PATCH 9/9] Update asserted verifier error messages --- test/integration/interrupt_resume_test.rb | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/test/integration/interrupt_resume_test.rb b/test/integration/interrupt_resume_test.rb index 695d0995..72d078f6 100644 --- a/test/integration/interrupt_resume_test.rb +++ b/test/integration/interrupt_resume_test.rb @@ -254,8 +254,11 @@ def test_interrupt_resume_inline_verifier_will_verify_entries_in_reverify_store assert_equal 1, incorrect_tables.length assert_equal "gftest.test_table_1", incorrect_tables.first - error_line = ghostferry.error_lines.last - assert error_line["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: #{chosen_id}") + error_message = ghostferry.error_lines.last["msg"] + predicate = "cutover verification failed for: gftest.test_table_1 [PKs: #{chosen_id}" + expectation = error_message.start_with?(predicate) + + assert expectation, "error message: #{error_message.inspect}, didn't start with #{predicate.inspect}" end def test_interrupt_resume_inline_verifier_will_verify_additional_rows_changed_on_source_during_interrupt @@ -299,7 +302,7 @@ def test_interrupt_resume_inline_verifier_will_verify_additional_rows_changed_on assert_equal "gftest.test_table_1", incorrect_tables.first error_message = ghostferry.error_lines.last["msg"] - predicate = "cutover verification failed for: gftest.test_table_1 [paginationKeys: #{chosen_id}" + predicate = "cutover verification failed for: gftest.test_table_1 [PKs: #{chosen_id}" expectation = error_message.start_with?(predicate) assert expectation, "error message: #{error_message.inspect}, didn't start with #{predicate.inspect}" @@ -669,6 +672,11 @@ def test_issue_149_corrupted assert verification_ran assert_equal ["#{DEFAULT_DB}.#{DEFAULT_TABLE}"], incorrect_tables - assert ghostferry.error_lines.last["msg"].start_with?("cutover verification failed for: gftest.test_table_1 [paginationKeys: #{id_to_change}") + + error_message = ghostferry.error_lines.last["msg"] + predicate = "cutover verification failed for: gftest.test_table_1 [PKs: #{id_to_change}" + expectation = error_message.start_with?(predicate) + + assert expectation, "error message: #{error_message.inspect}, didn't start with #{predicate.inspect}" end end