Bug 57500 - Introduce retry behavior for remote testing

git-svn-id: https://svn.apache.org/repos/asf/jmeter/trunk@1655969 13f79535-47bb-0310-9956-ffa450edef68

bin/jmeter.properties

@@ -197,6 +197,18 @@ remote_hosts=127.0.0.1
 # You may need to open Firewall port on the Controller machine
 #client.rmi.localport=0
 
+# When distributed test is starting, there may be several attempts to initialize
+# remote engines. By default, only single try is made. Increase following property
+# to make it retry for additional times
+#client.tries=1
+
+# If there is initialization retries, following property sets delay between attempts
+#client.retries_delay=5000
+
+# When all initialization tries was made, test will fail if some remote engines are failed
+# Set following property to true to ignore failed nodes and proceed with test 
+#client.continue_on_fail=false
+
 # To change the default port (1099) used to access the server:
 #server.rmi.port=1234
 

src/core/org/apache/jmeter/JMeter.java

@@ -31,7 +31,6 @@ import java.net.InetAddress;
 import java.net.MalformedURLException;
 import java.net.SocketException;
 import java.text.SimpleDateFormat;
-import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Date;
 import java.util.Enumeration;
@@ -57,6 +56,7 @@ import org.apache.jmeter.engine.ClientJMeterEngine;
 import org.apache.jmeter.engine.JMeterEngine;
 import org.apache.jmeter.engine.RemoteJMeterEngineImpl;
 import org.apache.jmeter.engine.StandardJMeterEngine;
+import org.apache.jmeter.engine.DistributedRunner;
 import org.apache.jmeter.exceptions.IllegalUserActionException;
 import org.apache.jmeter.gui.GuiPackage;
 import org.apache.jmeter.gui.MainFrame;
@@ -811,35 +811,16 @@ public class JMeter implements JMeterPlugin {
                 engines.add(engine);
             } else {
                 java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, ",");//$NON-NLS-1$
-                List<String> failingEngines = new ArrayList<String>(st.countTokens());
+                List<String> hosts = new LinkedList<String>();
                 while (st.hasMoreElements()) {
-                    String el = (String) st.nextElement();
+                    hosts.add((String) st.nextElement());
-                    println("Configuring remote engine for " + el);
-                    log.info("Configuring remote engine for " + el);
-                    JMeterEngine eng = doRemoteInit(el.trim(), tree);
-                    if (null != eng) {
-                        engines.add(eng);
-                    } else {
-                        failingEngines.add(el);
-                        println("Failed to configure "+el);
-                    }
                 }
-                if (engines.isEmpty()) {
+                
-                    println("No remote engines were started.");
+                DistributedRunner distributedRunner=new DistributedRunner(this.remoteProps);
-                    return;
+                distributedRunner.setStdout(System.out);
-                }
+                distributedRunner.setStdErr(System.err);
-                if(failingEngines.size()>0) {
+                distributedRunner.init(hosts, tree);
-                    throw new IllegalArgumentException("The following remote engines could not be configured:"+failingEngines);
+                distributedRunner.start();
-                }
-                println("Starting remote engines");
-                log.info("Starting remote engines");
-                long now=System.currentTimeMillis();
-                println("Starting the test @ "+new Date(now)+" ("+now+")");
-                for (JMeterEngine engine : engines) {
-                    engine.runTest();
-                }
-                println("Remote engines have been started");
-                log.info("Remote engines have been started");
             }
             startUdpDdaemon(engines);
         } catch (Exception e) {
@@ -929,22 +910,6 @@ public class JMeter implements JMeterPlugin {
         return rc;
     }
 
-    private JMeterEngine doRemoteInit(String hostName, HashTree testTree) {
-        JMeterEngine engine = null;
-        try {
-            engine = new ClientJMeterEngine(hostName);
-        } catch (Exception e) {
-            log.fatalError("Failure connecting to remote host: "+hostName, e);
-            System.err.println("Failure connecting to remote host: "+hostName+" "+e);
-            return null;
-        }
-        engine.configure(testTree);
-        if (!remoteProps.isEmpty()) {
-            engine.setProperties(remoteProps);
-        }
-        return engine;
-    }
-
     /*
      * Listen to test and handle tidyup after non-GUI test completes.
      * If running a remote test, then after waiting a few seconds for listeners to finish files,

src/core/org/apache/jmeter/engine/ClientJMeterEngine.java

@@ -197,4 +197,8 @@ public class ClientJMeterEngine implements JMeterEngine {
     public boolean isActive() {
         return true;
     }
+
+    public String getHost() {
+        return host;
+    }
 }

src/core/org/apache/jmeter/engine/DistributedRunner.java

@@ -0,0 +1,265 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jmeter.engine;
+
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.PrintStream;
+import java.net.MalformedURLException;
+import java.rmi.NotBoundException;
+import java.rmi.RemoteException;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import org.apache.jmeter.util.JMeterUtils;
+import org.apache.jorphan.collections.HashTree;
+import org.apache.jorphan.logging.LoggingManager;
+import org.apache.log.Logger;
+
+/**
+ * This class serves all responsibility of starting and stopping distributed tests.
+ * It was refactored from JMeter and RemoteStart classes to unify retry behavior.
+ *
+ * @see org.apache.jmeter.JMeter
+ * @see org.apache.jmeter.gui.action.RemoteStart
+ */
+public class DistributedRunner {
+    private static final Logger log = LoggingManager.getLoggerForClass();
+
+    public static final String RETRIES_NUMBER = "client.tries"; // $NON-NLS-1$
+    public static final String RETRIES_DELAY = "client.retries_delay"; // $NON-NLS-1$
+    public static final String CONTINUE_ON_FAIL = "client.continue_on_fail"; // $NON-NLS-1$
+
+    private final Properties remoteProps;
+    private final boolean continueOnFail;
+    private final int retriesDelay;
+    private final int retriesNumber;
+    private PrintStream stdout = new PrintStream(new SilentOutputStream());
+    private PrintStream stderr = new PrintStream(new SilentOutputStream());
+    private final Map<String, JMeterEngine> engines = new HashMap<String, JMeterEngine>();
+
+
+    public DistributedRunner() {
+        this(new Properties());
+    }
+
+    public DistributedRunner(Properties props) {
+        remoteProps = props;
+        retriesNumber = JMeterUtils.getPropDefault(RETRIES_NUMBER, 1);
+        continueOnFail = JMeterUtils.getPropDefault(CONTINUE_ON_FAIL, false);
+        retriesDelay = JMeterUtils.getPropDefault(RETRIES_DELAY, 5000);
+    }
+
+    public void init(List<String> addresses, HashTree tree) {
+        // converting list into mutable version
+        List<String> addrs = new LinkedList<String>(addresses);
+
+        for (int tryNo = 0; tryNo < retriesNumber; tryNo++) {
+            if (tryNo > 0) {
+                println("Following remote engines will retry configuring: " + addrs);
+                println("Pausing before retry for " + retriesDelay + "ms");
+                try {
+                    Thread.sleep(retriesDelay);
+                } catch (InterruptedException e) {
+                    throw new RuntimeException("Interrupted while initializing remote", e);
+                }
+            }
+
+            int idx = 0;
+            while (idx < addrs.size()) {
+                String address = addrs.get(idx);
+                println("Configuring remote engine: " + address);
+                JMeterEngine engine = getClientEngine(address.trim(), tree);
+                if (engine != null) {
+                    engines.put(address, engine);
+                    addrs.remove(address);
+                } else {
+                    println("Failed to configure " + address);
+                    idx++;
+                }
+            }
+
+            if (addrs.size() == 0) {
+                break;
+            }
+        }
+
+        if (addrs.size() > 0) {
+            String msg = "Following remote engines could not be configured:" + addrs;
+            if (!continueOnFail || engines.size() == 0) {
+                stop();
+                throw new RuntimeException(msg);
+            } else {
+                println(msg);
+                println("Continuing without failed engines...");
+            }
+        }
+    }
+
+    /**
+     * Starts a remote testing engines
+     *
+     * @param addresses list of the DNS names or IP addresses of the remote testing engines
+     */
+    public void start(List<String> addresses) {
+        println("Starting remote engines");
+        long now = System.currentTimeMillis();
+        println("Starting the test @ " + new Date(now) + " (" + now + ")");
+        for (String address : addresses) {
+            try {
+                if (engines.containsKey(address)) {
+                    engines.get(address).runTest();
+                } else {
+                    log.warn("Host not found in list of active engines: " + address);
+                }
+            } catch (IllegalStateException e) {
+                JMeterUtils.reportErrorToUser(e.getMessage(), JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
+            } catch (JMeterEngineException e) {
+                JMeterUtils.reportErrorToUser(e.getMessage(), JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
+            }
+        }
+        println("Remote engines have been started");
+    }
+
+    /**
+     * Start all engines that were previously initiated
+     */
+    public void start() {
+        List<String> addresses = new LinkedList<String>();
+        addresses.addAll(engines.keySet());
+        start(addresses);
+    }
+
+    public void stop(List<String> addresses) {
+        println("Stopping remote engines");
+        for (String address : addresses) {
+            try {
+                if (engines.containsKey(address)) {
+                    engines.get(address).stopTest(true);
+                } else {
+                    log.warn("Host not found in list of active engines: " + address);
+                }
+            } catch (RuntimeException e) {
+                errln("Failed to stop test on " + address, e);
+            }
+        }
+        println("Remote engines have been stopped");
+    }
+
+    /**
+     * Stop all engines that were previously initiated
+     */
+    public void stop() {
+        List<String> addresses = new LinkedList<String>();
+        addresses.addAll(engines.keySet());
+        stop(addresses);
+    }
+
+    public void shutdown(List<String> addresses) {
+        println("Shutting down remote engines");
+        for (String address : addresses) {
+            try {
+                if (engines.containsKey(address)) {
+                    engines.get(address).stopTest(false);
+                } else {
+                    log.warn("Host not found in list of active engines: " + address);
+                }
+
+            } catch (RuntimeException e) {
+                errln("Failed to shutdown test on " + address, e);
+            }
+        }
+        println("Remote engines have been shut down");
+    }
+
+    public void exit(List<String> addresses) {
+        println("Exiting remote engines");
+        for (String address : addresses) {
+            try {
+                if (engines.containsKey(address)) {
+                    engines.get(address).exit();
+                } else {
+                    log.warn("Host not found in list of active engines: " + address);
+                }
+            } catch (RuntimeException e) {
+                errln("Failed to exit on " + address, e);
+            }
+        }
+        println("Remote engines have been exited");
+    }
+
+    private JMeterEngine getClientEngine(String address, HashTree testTree) {
+        JMeterEngine engine;
+        try {
+            engine = createEngine(address);
+            engine.configure(testTree);
+            if (!remoteProps.isEmpty()) {
+                engine.setProperties(remoteProps);
+            }
+            return engine;
+        } catch (Exception ex) {
+            log.error("Failed to create engine at " + address, ex);
+            JMeterUtils.reportErrorToUser(ex.getMessage(),
+                    JMeterUtils.getResString("remote_error_init") + ": " + address); // $NON-NLS-1$ $NON-NLS-2$
+            return null;
+        }
+    }
+
+    /**
+     * A factory method that might be overridden for unit testing
+     *
+     * @param address address for engine
+     * @return engine instance
+     * @throws RemoteException
+     * @throws NotBoundException
+     * @throws MalformedURLException
+     */
+    protected JMeterEngine createEngine(String address) throws RemoteException, NotBoundException, MalformedURLException {
+        return new ClientJMeterEngine(address);
+    }
+
+    private void println(String s) {
+        log.info(s);
+        stdout.println(s);
+    }
+
+    private void errln(String s, Exception e) {
+        log.error(s, e);
+        stderr.println(s + ": ");
+        e.printStackTrace(stderr);
+    }
+
+    public void setStdout(PrintStream stdout) {
+        this.stdout = stdout;
+    }
+
+    public void setStdErr(PrintStream stdErr) {
+        this.stderr = stdErr;
+    }
+
+    private class SilentOutputStream extends OutputStream {
+        @Override
+        public void write(int b) throws IOException {
+            // enjoy the silence
+        }
+    }
+}

src/core/org/apache/jmeter/gui/action/RemoteStart.java

@@ -20,15 +20,14 @@ package org.apache.jmeter.gui.action;
 
 import java.awt.Component;
 import java.awt.event.ActionEvent;
-import java.util.HashMap;
+import java.util.Arrays;
 import java.util.HashSet;
-import java.util.Map;
+import java.util.LinkedList;
+import java.util.List;
 import java.util.Set;
-
+import java.util.StringTokenizer;
 import org.apache.jmeter.JMeter;
-import org.apache.jmeter.engine.ClientJMeterEngine;
+import org.apache.jmeter.engine.DistributedRunner;
-import org.apache.jmeter.engine.JMeterEngine;
-import org.apache.jmeter.engine.JMeterEngineException;
 import org.apache.jmeter.gui.GuiPackage;
 import org.apache.jmeter.threads.RemoteThreadsListenerTestElement;
 import org.apache.jmeter.util.JMeterUtils;
@@ -59,7 +58,7 @@ public class RemoteStart extends AbstractAction {
         commands.add(ActionNames.REMOTE_EXIT_ALL);
     }
 
-    private final Map<String, JMeterEngine> remoteEngines = new HashMap<String, JMeterEngine>();
+    private DistributedRunner distributedRunner = new DistributedRunner();
 
     public RemoteStart() {
     }
@@ -72,123 +71,37 @@ public class RemoteStart extends AbstractAction {
         }
         String action = e.getActionCommand();
         if (action.equals(ActionNames.REMOTE_STOP)) {
-            doRemoteStop(name, true);
+            GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
+            distributedRunner.stop(Arrays.asList(name));
         } else if (action.equals(ActionNames.REMOTE_SHUT)) {
-            doRemoteStop(name, false);
+            GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
+            distributedRunner.shutdown(Arrays.asList(name));
         } else if (action.equals(ActionNames.REMOTE_START)) {
             popupShouldSave(e);
-            doRemoteInit(name);
+            distributedRunner.init(Arrays.asList(name), getTestTree());
-            doRemoteStart(name);
+            distributedRunner.start(Arrays.asList(name));
         } else if (action.equals(ActionNames.REMOTE_START_ALL)) {
             popupShouldSave(e);
-            String remote_hosts_string = JMeterUtils.getPropDefault(REMOTE_HOSTS, LOCAL_HOST);
+            distributedRunner.init(getRemoteHosts(), getTestTree());
-            java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
+            distributedRunner.start();
-            while (st.hasMoreElements()) {
-                String el = (String) st.nextElement();
-                doRemoteInit(el.trim());
-            }
-            st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
-            while (st.hasMoreElements()) {
-                String el = (String) st.nextElement();
-                doRemoteStart(el.trim());
-            }
         } else if (action.equals(ActionNames.REMOTE_STOP_ALL)) {
-            doRemoteStopAll(true);
+            distributedRunner.stop(getRemoteHosts());
         } else if (action.equals(ActionNames.REMOTE_SHUT_ALL)) {
-            doRemoteStopAll(false);
+            distributedRunner.shutdown(getRemoteHosts());
         } else if (action.equals(ActionNames.REMOTE_EXIT)) {
-            doRemoteExit(name);
+            distributedRunner.exit(Arrays.asList(name));
         } else if (action.equals(ActionNames.REMOTE_EXIT_ALL)) {
-            String remote_hosts_string = JMeterUtils.getPropDefault(REMOTE_HOSTS, LOCAL_HOST);
+            distributedRunner.exit(getRemoteHosts());
-            java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
-            while (st.hasMoreElements()) {
-                String el = (String) st.nextElement();
-                doRemoteExit(el.trim());
-            }
         }
     }
 
-    private void doRemoteStopAll(boolean now) {
+    private List<String> getRemoteHosts() {
         String remote_hosts_string = JMeterUtils.getPropDefault(REMOTE_HOSTS, LOCAL_HOST);
-        java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
+        StringTokenizer st = new StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
-        while (st.hasMoreElements()) {
+        List<String> list = new LinkedList<String>();
-            String el = (String) st.nextElement();
+        while (st.hasMoreElements())
-            doRemoteStop(el.trim(), now);
+            list.add((String) st.nextElement());
-        }
+        return list;
-    }
-
-    /**
-     * Stops a remote testing engine
-     *
-     * @param name
-     *            the DNS name or IP address of the remote testing engine
-     *
-     */
-    private void doRemoteStop(String name, boolean now) {
-        GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
-        JMeterEngine engine = remoteEngines.get(name);
-        // Engine may be null if it has not correctly started
-        if(engine != null) {
-            engine.stopTest(now);
-        }
-    }
-
-    /**
-     * Exits a remote testing engine
-     *
-     * @param name
-     *            the DNS name or IP address of the remote testing engine
-     *
-     */
-    private void doRemoteExit(String name) {
-        JMeterEngine engine = remoteEngines.get(name);
-        if (engine == null) {
-            return;
-        }
-        // GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
-        engine.exit();
-    }
-
-    /**
-     * Starts a remote testing engine
-     *
-     * @param name
-     *            the DNS name or IP address of the remote testing engine
-     *
-     */
-    private void doRemoteStart(String name) {
-        JMeterEngine engine = remoteEngines.get(name);
-        if (engine != null) {
-            try {
-                engine.runTest();
-            } catch (IllegalStateException e) {
-                JMeterUtils.reportErrorToUser(e.getMessage(),JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
-            } catch (JMeterEngineException e) {
-                JMeterUtils.reportErrorToUser(e.getMessage(),JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
-            }
-        }
-    }
-
-    /**
-     * Initializes remote engines
-     */
-    private void doRemoteInit(String name) {
-        JMeterEngine engine = remoteEngines.get(name);
-        if (engine == null) {
-            try {
-                log.info("Initialising remote engine: "+name);
-                engine = new ClientJMeterEngine(name);
-                remoteEngines.put(name, engine);
-            } catch (Exception ex) {
-                log.error("Failed to initialise remote engine", ex);
-                JMeterUtils.reportErrorToUser(ex.getMessage(),
-                        JMeterUtils.getResString("remote_error_init") + ": " + name); // $NON-NLS-1$ $NON-NLS-2$
-                return;
-            }
-        } else {
-            engine.reset();
-        }
-        initEngine(engine);
     }
 
     @Override
@@ -196,19 +109,13 @@ public class RemoteStart extends AbstractAction {
         return commands;
     }
 
-    /**
+    private HashTree getTestTree() {
-     * Initializes test on engine.
-     *
-     * @param engine
-     *            remote engine object
-     */
-    private void initEngine(JMeterEngine engine) {
         GuiPackage gui = GuiPackage.getInstance();
         HashTree testTree = gui.getTreeModel().getTestPlan();
         JMeter.convertSubTree(testTree);
         testTree.add(testTree.getArray()[0], gui.getMainFrame());
         // Used for remote notification of threads start/stop,see BUG 54152
         testTree.add(testTree.getArray()[0], new RemoteThreadsListenerTestElement());
-        engine.configure(testTree);
+        return testTree;
     }
 }

test/src/org/apache/jmeter/engine/DistributedRunnerTest.java

@@ -0,0 +1,166 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.jmeter.engine;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.MalformedURLException;
+import java.rmi.NotBoundException;
+import java.rmi.RemoteException;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Properties;
+import org.apache.jmeter.util.JMeterUtils;
+import org.apache.jorphan.collections.HashTree;
+import org.apache.jorphan.logging.LoggingManager;
+import org.apache.log.Logger;
+
+public class DistributedRunnerTest extends junit.framework.TestCase {
+
+    public static void createJmeterEnv() throws IOException {
+        File propsFile;
+        try {
+            propsFile = File.createTempFile("jmeter-plugins", ".properties");
+            propsFile.deleteOnExit();
+            JMeterUtils.loadJMeterProperties(propsFile.getAbsolutePath());
+        } catch (IOException ex) {
+            ex.printStackTrace(System.err);
+        }
+        JMeterUtils.setLocale(new Locale("ignoreResources"));
+    }
+
+    public void testSuccess() throws Exception {
+        createJmeterEnv();
+        JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "1");
+        JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "false");
+        DistributedRunnerEmul obj = new DistributedRunnerEmul();
+        obj.engines.add(new EmulatorEngine());
+        obj.engines.add(new EmulatorEngine());
+        List<String> hosts = Arrays.asList("test1", "test2");
+        obj.init(hosts, new HashTree());
+        obj.start();
+        obj.shutdown(hosts);
+        obj.stop(hosts);
+        obj.exit(hosts);
+    }
+
+    public void testFailure1() throws Exception {
+        createJmeterEnv();
+        JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "2");
+        JMeterUtils.setProperty(DistributedRunner.RETRIES_DELAY, "1");
+        JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "true");
+        DistributedRunnerEmul obj = new DistributedRunnerEmul();
+        List<String> hosts = Arrays.asList("test1", "test2");
+        obj.init(hosts, new HashTree());
+        obj.start();
+        obj.shutdown(hosts);
+        obj.stop(hosts);
+        obj.exit(hosts);
+    }
+
+    public void testFailure2() throws Exception {
+        createJmeterEnv();
+        JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "1");
+        JMeterUtils.setProperty(DistributedRunner.RETRIES_DELAY, "1");
+        JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "false");
+        DistributedRunnerEmul obj = new DistributedRunnerEmul();
+        List<String> hosts = Arrays.asList("test1", "test2");
+        try {
+            obj.init(hosts, new HashTree());
+            fail();
+        } catch (RuntimeException ignored) {
+        }
+    }
+
+    public void testFailure3() throws Exception {
+        createJmeterEnv();
+        JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "1");
+        JMeterUtils.setProperty(DistributedRunner.RETRIES_DELAY, "1");
+        JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "true");
+        DistributedRunnerEmul obj = new DistributedRunnerEmul();
+        List<String> hosts = Arrays.asList("test1", "test2");
+        obj.init(hosts, new HashTree());
+        obj.start(hosts);
+        obj.shutdown(hosts);
+        obj.stop(hosts);
+        obj.exit(hosts);
+    }
+
+    private class DistributedRunnerEmul extends DistributedRunner {
+        public List<EmulatorEngine> engines = new LinkedList<EmulatorEngine>();
+
+        @Override
+        protected JMeterEngine createEngine(String address) throws RemoteException, NotBoundException, MalformedURLException {
+            EmulatorEngine engine = engines.remove(0);
+            engine.setHost(address);
+            return engine;
+        }
+    }
+
+    private static class EmulatorEngine implements JMeterEngine {
+        private static final Logger log = LoggingManager.getLoggerForClass();
+        private String host;
+
+        public EmulatorEngine() {
+            log.debug("Creating emulator " + host);
+        }
+
+        @Override
+        public void configure(HashTree testPlan) {
+            log.debug("Configuring " + host);
+        }
+
+        @Override
+        public void runTest() throws JMeterEngineException {
+            log.debug("Running " + host);
+        }
+
+        @Override
+        public void stopTest(boolean now) {
+            log.debug("Stopping " + host);
+        }
+
+        @Override
+        public void reset() {
+            log.debug("Resetting " + host);
+        }
+
+        @Override
+        public void setProperties(Properties p) {
+            log.debug("Set properties " + host);
+        }
+
+        @Override
+        public void exit() {
+            log.debug("Exitting " + host);
+        }
+
+        @Override
+        public boolean isActive() {
+            log.debug("Check if active " + host);
+            return false;
+        }
+
+        public void setHost(String host) {
+            this.host = host;
+        }
+    }
+}

xdocs/changes.xml

@@ -56,6 +56,7 @@ Summary
 <p>
 <ul>
 <li><bugzilla>48799</bugzilla> - Add time to establish connection to available sample metrics. Implemented by Andrey Pokhilko (andrey at blazemeter.com) and contributed by BlazeMeter Ltd.</li>
+<li><bugzilla>57500</bugzilla> - Introduce retry behavior for distributed testing. Implemented by Andrey Pokhilko and Dzimitry Kashlach and contributed by BlazeMeter Ltd.</li>
 <li>Sample text</li>
 </ul>
 </p>
@@ -243,6 +244,7 @@ See  <bugzilla>56357</bugzilla> for details.
 <li><a href="http://ubikloadpack.com">Ubik Load Pack</a></li>
 <li>Yngvi &amp;THORN;&amp;oacute;r Sigurj&amp;oacute;nsson (blitzkopf at gmail.com)</li>
 <li>Dzmitry Kashlach (dzmitrykashlach at gmail.com)</li>
+<li><a href="http://blazemeter.com">BlazeMeter Ltd.</a></li>
 <li>Benoit Wiart (benoit.wiart at gmail.com)</li>
 </ul>
 

xdocs/usermanual/remote-test.xml

@@ -282,7 +282,24 @@ There are some JMeter properties that can be set to alter this behaviour.
 </subsection>
 
 
-
+<subsection name="&sect-num;.5 Dealing with nodes that failed starting" anchor="retries">
+  <p>
+    For a large-scale tests there is a chance that some part of remote servers will be unavailable or down. 
+    For example, when you use automation script to allocate many cloud machines and use them as generators,
+    some of requested machines might fail booting because of cloud's issues. 
+  </p>
+  <p>
+    First what you might want is to retry initialization attempts in hope that failed nodes just slightly delayed their boot. 
+    To enable retries, you should set <code>client.tries</code> property to total number of connection attempts. 
+    By default it does only one attempt. To control retry delay, set the <code>client.retries_delay</code> property
+    to number of milliseconds to sleep between attempts.
+  </p>
+    
+  <p>
+    Finally, you might still want to run the test with those generators that succeeded initialization and skipping failed nodes.
+    To enable that, set the <code>client.continue_on_fail=true</code> property.
+  </p>
+</subsection>
 
 </section>