kafka-883; System Test - update migration tool testsuite after 0.7 ProducerPerformance sends seq MessageID; patched by John Fung; reviewed by Jun Rao

system_test/migration_tool_testsuite/config/migration_producer.properties

@@ -61,3 +61,6 @@ serializer.class=kafka.serializer.DefaultEncoder
 
 # the number of messages batched at the producer 
 #batch.num.messages=
+
+message.send.max.retries=3
+request.required.acks=1

system_test/migration_tool_testsuite/migration_tool_test.py

@@ -76,6 +76,8 @@ class MigrationToolTest(ReplicationUtils, SetupUtils):
             self.testSuiteAbsPathName, SystemTestEnv.SYSTEM_TEST_CASE_PREFIX)
         testCasePathNameList.sort()
 
+        replicationUtils = ReplicationUtils(self)
+
         # =============================================================
         # launch each testcase one by one: testcase_1, testcase_2, ...
         # =============================================================
@@ -284,7 +286,7 @@ class MigrationToolTest(ReplicationUtils, SetupUtils):
                 # validate the data matched and checksum
                 # =============================================
                 self.log_message("validating data matched")
-                kafka_system_test_utils.validate_07_08_migrated_data_matched(self.systemTestEnv, self.testcaseEnv)
+                kafka_system_test_utils.validate_data_matched(self.systemTestEnv, self.testcaseEnv, replicationUtils)
                 kafka_system_test_utils.validate_broker_log_segment_checksum(self.systemTestEnv, self.testcaseEnv)
 
                 # =============================================

system_test/migration_tool_testsuite/testcase_9001/testcase_9001_properties.json

@@ -100,7 +100,6 @@
       "group.id": "mytestgroup",
       "consumer-timeout-ms": "10000",
       "log_filename": "console_consumer_8.log",
-      "formatter": "kafka.consumer.ChecksumMessageFormatter",
       "config_filename": "console_consumer_8.properties"
     },
     {

system_test/migration_tool_testsuite/testcase_9003/testcase_9003_properties.json

@@ -101,7 +101,6 @@
       "group.id": "mytestgroup",
       "consumer-timeout-ms": "10000",
       "log_filename": "console_consumer_8.log",
-      "formatter": "kafka.consumer.ChecksumMessageFormatter",
       "config_filename": "console_consumer_8.properties"
     },
     {

system_test/migration_tool_testsuite/testcase_9004/testcase_9004_properties.json

@@ -101,7 +101,6 @@
       "group.id": "mytestgroup",
       "consumer-timeout-ms": "10000",
       "log_filename": "console_consumer_8.log",
-      "formatter": "kafka.consumer.ChecksumMessageFormatter",
       "config_filename": "console_consumer_8.properties"
     },
     {

system_test/migration_tool_testsuite/testcase_9005/testcase_9005_properties.json

@@ -128,7 +128,6 @@
       "group.id": "mytestgroup",
       "consumer-timeout-ms": "10000",
       "log_filename": "console_consumer_10.log",
-      "formatter": "kafka.consumer.ChecksumMessageFormatter",
       "config_filename": "console_consumer_10.properties"
     },
     {
@@ -137,7 +136,6 @@
       "group.id": "mytestgroup",
       "consumer-timeout-ms": "10000",
       "log_filename": "console_consumer_11.log",
-      "formatter": "kafka.consumer.ChecksumMessageFormatter",
       "config_filename": "console_consumer_11.properties"
     },
 

system_test/migration_tool_testsuite/testcase_9006/testcase_9006_properties.json

@@ -128,7 +128,6 @@
       "group.id": "mytestgroup",
       "consumer-timeout-ms": "10000",
       "log_filename": "console_consumer_10.log",
-      "formatter": "kafka.consumer.ChecksumMessageFormatter",
       "config_filename": "console_consumer_10.properties"
     },
     {
@@ -137,7 +136,6 @@
       "group.id": "mytestgroup",
       "consumer-timeout-ms": "10000",
       "log_filename": "console_consumer_11.log",
-      "formatter": "kafka.consumer.ChecksumMessageFormatter",
       "config_filename": "console_consumer_11.properties"
     },
 

system_test/utils/kafka_system_test_utils.py

@@ -1047,6 +1047,7 @@ def start_producer_in_thread(testcaseEnv, entityConfigList, producerConfig, kafk
                        "JMX_PORT=" + jmxPort,
                        kafkaRunClassBin + " kafka.perf.ProducerPerformance",
                        "--brokerinfo " + brokerInfoStr,
+                       "--initial-message-id " + str(initMsgId),
                        "--messages " + noMsgPerBatch,
                        "--topic " + topic,
                        "--threads " + threads,