Bug 57500 - Introduce retry behavior for remote testing

git-svn-id: https://svn.apache.org/repos/asf/jmeter/trunk@1655969 13f79535-47bb-0310-9956-ffa450edef68
This commit is contained in:
Andrey Pokhilko 2015-01-30 09:33:21 +00:00
parent d494ee73b3
commit 022af006bf
8 changed files with 502 additions and 164 deletions

View File

@ -197,6 +197,18 @@ remote_hosts=127.0.0.1
# You may need to open Firewall port on the Controller machine
#client.rmi.localport=0
# When distributed test is starting, there may be several attempts to initialize
# remote engines. By default, only single try is made. Increase following property
# to make it retry for additional times
#client.tries=1
# If there is initialization retries, following property sets delay between attempts
#client.retries_delay=5000
# When all initialization tries was made, test will fail if some remote engines are failed
# Set following property to true to ignore failed nodes and proceed with test
#client.continue_on_fail=false
# To change the default port (1099) used to access the server:
#server.rmi.port=1234

View File

@ -31,7 +31,6 @@ import java.net.InetAddress;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Date;
import java.util.Enumeration;
@ -57,6 +56,7 @@ import org.apache.jmeter.engine.ClientJMeterEngine;
import org.apache.jmeter.engine.JMeterEngine;
import org.apache.jmeter.engine.RemoteJMeterEngineImpl;
import org.apache.jmeter.engine.StandardJMeterEngine;
import org.apache.jmeter.engine.DistributedRunner;
import org.apache.jmeter.exceptions.IllegalUserActionException;
import org.apache.jmeter.gui.GuiPackage;
import org.apache.jmeter.gui.MainFrame;
@ -811,35 +811,16 @@ public class JMeter implements JMeterPlugin {
engines.add(engine);
} else {
java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, ",");//$NON-NLS-1$
List<String> failingEngines = new ArrayList<String>(st.countTokens());
List<String> hosts = new LinkedList<String>();
while (st.hasMoreElements()) {
String el = (String) st.nextElement();
println("Configuring remote engine for " + el);
log.info("Configuring remote engine for " + el);
JMeterEngine eng = doRemoteInit(el.trim(), tree);
if (null != eng) {
engines.add(eng);
} else {
failingEngines.add(el);
println("Failed to configure "+el);
}
hosts.add((String) st.nextElement());
}
if (engines.isEmpty()) {
println("No remote engines were started.");
return;
}
if(failingEngines.size()>0) {
throw new IllegalArgumentException("The following remote engines could not be configured:"+failingEngines);
}
println("Starting remote engines");
log.info("Starting remote engines");
long now=System.currentTimeMillis();
println("Starting the test @ "+new Date(now)+" ("+now+")");
for (JMeterEngine engine : engines) {
engine.runTest();
}
println("Remote engines have been started");
log.info("Remote engines have been started");
DistributedRunner distributedRunner=new DistributedRunner(this.remoteProps);
distributedRunner.setStdout(System.out);
distributedRunner.setStdErr(System.err);
distributedRunner.init(hosts, tree);
distributedRunner.start();
}
startUdpDdaemon(engines);
} catch (Exception e) {
@ -929,22 +910,6 @@ public class JMeter implements JMeterPlugin {
return rc;
}
private JMeterEngine doRemoteInit(String hostName, HashTree testTree) {
JMeterEngine engine = null;
try {
engine = new ClientJMeterEngine(hostName);
} catch (Exception e) {
log.fatalError("Failure connecting to remote host: "+hostName, e);
System.err.println("Failure connecting to remote host: "+hostName+" "+e);
return null;
}
engine.configure(testTree);
if (!remoteProps.isEmpty()) {
engine.setProperties(remoteProps);
}
return engine;
}
/*
* Listen to test and handle tidyup after non-GUI test completes.
* If running a remote test, then after waiting a few seconds for listeners to finish files,

View File

@ -197,4 +197,8 @@ public class ClientJMeterEngine implements JMeterEngine {
public boolean isActive() {
return true;
}
public String getHost() {
return host;
}
}

View File

@ -0,0 +1,265 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.jmeter.engine;
import java.io.IOException;
import java.io.OutputStream;
import java.io.PrintStream;
import java.net.MalformedURLException;
import java.rmi.NotBoundException;
import java.rmi.RemoteException;
import java.util.Date;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import org.apache.jmeter.util.JMeterUtils;
import org.apache.jorphan.collections.HashTree;
import org.apache.jorphan.logging.LoggingManager;
import org.apache.log.Logger;
/**
* This class serves all responsibility of starting and stopping distributed tests.
* It was refactored from JMeter and RemoteStart classes to unify retry behavior.
*
* @see org.apache.jmeter.JMeter
* @see org.apache.jmeter.gui.action.RemoteStart
*/
public class DistributedRunner {
private static final Logger log = LoggingManager.getLoggerForClass();
public static final String RETRIES_NUMBER = "client.tries"; // $NON-NLS-1$
public static final String RETRIES_DELAY = "client.retries_delay"; // $NON-NLS-1$
public static final String CONTINUE_ON_FAIL = "client.continue_on_fail"; // $NON-NLS-1$
private final Properties remoteProps;
private final boolean continueOnFail;
private final int retriesDelay;
private final int retriesNumber;
private PrintStream stdout = new PrintStream(new SilentOutputStream());
private PrintStream stderr = new PrintStream(new SilentOutputStream());
private final Map<String, JMeterEngine> engines = new HashMap<String, JMeterEngine>();
public DistributedRunner() {
this(new Properties());
}
public DistributedRunner(Properties props) {
remoteProps = props;
retriesNumber = JMeterUtils.getPropDefault(RETRIES_NUMBER, 1);
continueOnFail = JMeterUtils.getPropDefault(CONTINUE_ON_FAIL, false);
retriesDelay = JMeterUtils.getPropDefault(RETRIES_DELAY, 5000);
}
public void init(List<String> addresses, HashTree tree) {
// converting list into mutable version
List<String> addrs = new LinkedList<String>(addresses);
for (int tryNo = 0; tryNo < retriesNumber; tryNo++) {
if (tryNo > 0) {
println("Following remote engines will retry configuring: " + addrs);
println("Pausing before retry for " + retriesDelay + "ms");
try {
Thread.sleep(retriesDelay);
} catch (InterruptedException e) {
throw new RuntimeException("Interrupted while initializing remote", e);
}
}
int idx = 0;
while (idx < addrs.size()) {
String address = addrs.get(idx);
println("Configuring remote engine: " + address);
JMeterEngine engine = getClientEngine(address.trim(), tree);
if (engine != null) {
engines.put(address, engine);
addrs.remove(address);
} else {
println("Failed to configure " + address);
idx++;
}
}
if (addrs.size() == 0) {
break;
}
}
if (addrs.size() > 0) {
String msg = "Following remote engines could not be configured:" + addrs;
if (!continueOnFail || engines.size() == 0) {
stop();
throw new RuntimeException(msg);
} else {
println(msg);
println("Continuing without failed engines...");
}
}
}
/**
* Starts a remote testing engines
*
* @param addresses list of the DNS names or IP addresses of the remote testing engines
*/
public void start(List<String> addresses) {
println("Starting remote engines");
long now = System.currentTimeMillis();
println("Starting the test @ " + new Date(now) + " (" + now + ")");
for (String address : addresses) {
try {
if (engines.containsKey(address)) {
engines.get(address).runTest();
} else {
log.warn("Host not found in list of active engines: " + address);
}
} catch (IllegalStateException e) {
JMeterUtils.reportErrorToUser(e.getMessage(), JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
} catch (JMeterEngineException e) {
JMeterUtils.reportErrorToUser(e.getMessage(), JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
}
}
println("Remote engines have been started");
}
/**
* Start all engines that were previously initiated
*/
public void start() {
List<String> addresses = new LinkedList<String>();
addresses.addAll(engines.keySet());
start(addresses);
}
public void stop(List<String> addresses) {
println("Stopping remote engines");
for (String address : addresses) {
try {
if (engines.containsKey(address)) {
engines.get(address).stopTest(true);
} else {
log.warn("Host not found in list of active engines: " + address);
}
} catch (RuntimeException e) {
errln("Failed to stop test on " + address, e);
}
}
println("Remote engines have been stopped");
}
/**
* Stop all engines that were previously initiated
*/
public void stop() {
List<String> addresses = new LinkedList<String>();
addresses.addAll(engines.keySet());
stop(addresses);
}
public void shutdown(List<String> addresses) {
println("Shutting down remote engines");
for (String address : addresses) {
try {
if (engines.containsKey(address)) {
engines.get(address).stopTest(false);
} else {
log.warn("Host not found in list of active engines: " + address);
}
} catch (RuntimeException e) {
errln("Failed to shutdown test on " + address, e);
}
}
println("Remote engines have been shut down");
}
public void exit(List<String> addresses) {
println("Exiting remote engines");
for (String address : addresses) {
try {
if (engines.containsKey(address)) {
engines.get(address).exit();
} else {
log.warn("Host not found in list of active engines: " + address);
}
} catch (RuntimeException e) {
errln("Failed to exit on " + address, e);
}
}
println("Remote engines have been exited");
}
private JMeterEngine getClientEngine(String address, HashTree testTree) {
JMeterEngine engine;
try {
engine = createEngine(address);
engine.configure(testTree);
if (!remoteProps.isEmpty()) {
engine.setProperties(remoteProps);
}
return engine;
} catch (Exception ex) {
log.error("Failed to create engine at " + address, ex);
JMeterUtils.reportErrorToUser(ex.getMessage(),
JMeterUtils.getResString("remote_error_init") + ": " + address); // $NON-NLS-1$ $NON-NLS-2$
return null;
}
}
/**
* A factory method that might be overridden for unit testing
*
* @param address address for engine
* @return engine instance
* @throws RemoteException
* @throws NotBoundException
* @throws MalformedURLException
*/
protected JMeterEngine createEngine(String address) throws RemoteException, NotBoundException, MalformedURLException {
return new ClientJMeterEngine(address);
}
private void println(String s) {
log.info(s);
stdout.println(s);
}
private void errln(String s, Exception e) {
log.error(s, e);
stderr.println(s + ": ");
e.printStackTrace(stderr);
}
public void setStdout(PrintStream stdout) {
this.stdout = stdout;
}
public void setStdErr(PrintStream stdErr) {
this.stderr = stdErr;
}
private class SilentOutputStream extends OutputStream {
@Override
public void write(int b) throws IOException {
// enjoy the silence
}
}
}

View File

@ -20,15 +20,14 @@ package org.apache.jmeter.gui.action;
import java.awt.Component;
import java.awt.event.ActionEvent;
import java.util.HashMap;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Map;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import java.util.StringTokenizer;
import org.apache.jmeter.JMeter;
import org.apache.jmeter.engine.ClientJMeterEngine;
import org.apache.jmeter.engine.JMeterEngine;
import org.apache.jmeter.engine.JMeterEngineException;
import org.apache.jmeter.engine.DistributedRunner;
import org.apache.jmeter.gui.GuiPackage;
import org.apache.jmeter.threads.RemoteThreadsListenerTestElement;
import org.apache.jmeter.util.JMeterUtils;
@ -59,7 +58,7 @@ public class RemoteStart extends AbstractAction {
commands.add(ActionNames.REMOTE_EXIT_ALL);
}
private final Map<String, JMeterEngine> remoteEngines = new HashMap<String, JMeterEngine>();
private DistributedRunner distributedRunner = new DistributedRunner();
public RemoteStart() {
}
@ -72,123 +71,37 @@ public class RemoteStart extends AbstractAction {
}
String action = e.getActionCommand();
if (action.equals(ActionNames.REMOTE_STOP)) {
doRemoteStop(name, true);
GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
distributedRunner.stop(Arrays.asList(name));
} else if (action.equals(ActionNames.REMOTE_SHUT)) {
doRemoteStop(name, false);
GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
distributedRunner.shutdown(Arrays.asList(name));
} else if (action.equals(ActionNames.REMOTE_START)) {
popupShouldSave(e);
doRemoteInit(name);
doRemoteStart(name);
distributedRunner.init(Arrays.asList(name), getTestTree());
distributedRunner.start(Arrays.asList(name));
} else if (action.equals(ActionNames.REMOTE_START_ALL)) {
popupShouldSave(e);
String remote_hosts_string = JMeterUtils.getPropDefault(REMOTE_HOSTS, LOCAL_HOST);
java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
while (st.hasMoreElements()) {
String el = (String) st.nextElement();
doRemoteInit(el.trim());
}
st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
while (st.hasMoreElements()) {
String el = (String) st.nextElement();
doRemoteStart(el.trim());
}
distributedRunner.init(getRemoteHosts(), getTestTree());
distributedRunner.start();
} else if (action.equals(ActionNames.REMOTE_STOP_ALL)) {
doRemoteStopAll(true);
distributedRunner.stop(getRemoteHosts());
} else if (action.equals(ActionNames.REMOTE_SHUT_ALL)) {
doRemoteStopAll(false);
distributedRunner.shutdown(getRemoteHosts());
} else if (action.equals(ActionNames.REMOTE_EXIT)) {
doRemoteExit(name);
distributedRunner.exit(Arrays.asList(name));
} else if (action.equals(ActionNames.REMOTE_EXIT_ALL)) {
String remote_hosts_string = JMeterUtils.getPropDefault(REMOTE_HOSTS, LOCAL_HOST);
java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
while (st.hasMoreElements()) {
String el = (String) st.nextElement();
doRemoteExit(el.trim());
}
distributedRunner.exit(getRemoteHosts());
}
}
private void doRemoteStopAll(boolean now) {
private List<String> getRemoteHosts() {
String remote_hosts_string = JMeterUtils.getPropDefault(REMOTE_HOSTS, LOCAL_HOST);
java.util.StringTokenizer st = new java.util.StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
while (st.hasMoreElements()) {
String el = (String) st.nextElement();
doRemoteStop(el.trim(), now);
}
}
/**
* Stops a remote testing engine
*
* @param name
* the DNS name or IP address of the remote testing engine
*
*/
private void doRemoteStop(String name, boolean now) {
GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
JMeterEngine engine = remoteEngines.get(name);
// Engine may be null if it has not correctly started
if(engine != null) {
engine.stopTest(now);
}
}
/**
* Exits a remote testing engine
*
* @param name
* the DNS name or IP address of the remote testing engine
*
*/
private void doRemoteExit(String name) {
JMeterEngine engine = remoteEngines.get(name);
if (engine == null) {
return;
}
// GuiPackage.getInstance().getMainFrame().showStoppingMessage(name);
engine.exit();
}
/**
* Starts a remote testing engine
*
* @param name
* the DNS name or IP address of the remote testing engine
*
*/
private void doRemoteStart(String name) {
JMeterEngine engine = remoteEngines.get(name);
if (engine != null) {
try {
engine.runTest();
} catch (IllegalStateException e) {
JMeterUtils.reportErrorToUser(e.getMessage(),JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
} catch (JMeterEngineException e) {
JMeterUtils.reportErrorToUser(e.getMessage(),JMeterUtils.getResString("remote_error_starting")); // $NON-NLS-1$
}
}
}
/**
* Initializes remote engines
*/
private void doRemoteInit(String name) {
JMeterEngine engine = remoteEngines.get(name);
if (engine == null) {
try {
log.info("Initialising remote engine: "+name);
engine = new ClientJMeterEngine(name);
remoteEngines.put(name, engine);
} catch (Exception ex) {
log.error("Failed to initialise remote engine", ex);
JMeterUtils.reportErrorToUser(ex.getMessage(),
JMeterUtils.getResString("remote_error_init") + ": " + name); // $NON-NLS-1$ $NON-NLS-2$
return;
}
} else {
engine.reset();
}
initEngine(engine);
StringTokenizer st = new StringTokenizer(remote_hosts_string, REMOTE_HOSTS_SEPARATOR);
List<String> list = new LinkedList<String>();
while (st.hasMoreElements())
list.add((String) st.nextElement());
return list;
}
@Override
@ -196,19 +109,13 @@ public class RemoteStart extends AbstractAction {
return commands;
}
/**
* Initializes test on engine.
*
* @param engine
* remote engine object
*/
private void initEngine(JMeterEngine engine) {
private HashTree getTestTree() {
GuiPackage gui = GuiPackage.getInstance();
HashTree testTree = gui.getTreeModel().getTestPlan();
JMeter.convertSubTree(testTree);
testTree.add(testTree.getArray()[0], gui.getMainFrame());
// Used for remote notification of threads start/stop,see BUG 54152
testTree.add(testTree.getArray()[0], new RemoteThreadsListenerTestElement());
engine.configure(testTree);
return testTree;
}
}

View File

@ -0,0 +1,166 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.jmeter.engine;
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.rmi.NotBoundException;
import java.rmi.RemoteException;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.Locale;
import java.util.Properties;
import org.apache.jmeter.util.JMeterUtils;
import org.apache.jorphan.collections.HashTree;
import org.apache.jorphan.logging.LoggingManager;
import org.apache.log.Logger;
public class DistributedRunnerTest extends junit.framework.TestCase {
public static void createJmeterEnv() throws IOException {
File propsFile;
try {
propsFile = File.createTempFile("jmeter-plugins", ".properties");
propsFile.deleteOnExit();
JMeterUtils.loadJMeterProperties(propsFile.getAbsolutePath());
} catch (IOException ex) {
ex.printStackTrace(System.err);
}
JMeterUtils.setLocale(new Locale("ignoreResources"));
}
public void testSuccess() throws Exception {
createJmeterEnv();
JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "1");
JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "false");
DistributedRunnerEmul obj = new DistributedRunnerEmul();
obj.engines.add(new EmulatorEngine());
obj.engines.add(new EmulatorEngine());
List<String> hosts = Arrays.asList("test1", "test2");
obj.init(hosts, new HashTree());
obj.start();
obj.shutdown(hosts);
obj.stop(hosts);
obj.exit(hosts);
}
public void testFailure1() throws Exception {
createJmeterEnv();
JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "2");
JMeterUtils.setProperty(DistributedRunner.RETRIES_DELAY, "1");
JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "true");
DistributedRunnerEmul obj = new DistributedRunnerEmul();
List<String> hosts = Arrays.asList("test1", "test2");
obj.init(hosts, new HashTree());
obj.start();
obj.shutdown(hosts);
obj.stop(hosts);
obj.exit(hosts);
}
public void testFailure2() throws Exception {
createJmeterEnv();
JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "1");
JMeterUtils.setProperty(DistributedRunner.RETRIES_DELAY, "1");
JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "false");
DistributedRunnerEmul obj = new DistributedRunnerEmul();
List<String> hosts = Arrays.asList("test1", "test2");
try {
obj.init(hosts, new HashTree());
fail();
} catch (RuntimeException ignored) {
}
}
public void testFailure3() throws Exception {
createJmeterEnv();
JMeterUtils.setProperty(DistributedRunner.RETRIES_NUMBER, "1");
JMeterUtils.setProperty(DistributedRunner.RETRIES_DELAY, "1");
JMeterUtils.setProperty(DistributedRunner.CONTINUE_ON_FAIL, "true");
DistributedRunnerEmul obj = new DistributedRunnerEmul();
List<String> hosts = Arrays.asList("test1", "test2");
obj.init(hosts, new HashTree());
obj.start(hosts);
obj.shutdown(hosts);
obj.stop(hosts);
obj.exit(hosts);
}
private class DistributedRunnerEmul extends DistributedRunner {
public List<EmulatorEngine> engines = new LinkedList<EmulatorEngine>();
@Override
protected JMeterEngine createEngine(String address) throws RemoteException, NotBoundException, MalformedURLException {
EmulatorEngine engine = engines.remove(0);
engine.setHost(address);
return engine;
}
}
private static class EmulatorEngine implements JMeterEngine {
private static final Logger log = LoggingManager.getLoggerForClass();
private String host;
public EmulatorEngine() {
log.debug("Creating emulator " + host);
}
@Override
public void configure(HashTree testPlan) {
log.debug("Configuring " + host);
}
@Override
public void runTest() throws JMeterEngineException {
log.debug("Running " + host);
}
@Override
public void stopTest(boolean now) {
log.debug("Stopping " + host);
}
@Override
public void reset() {
log.debug("Resetting " + host);
}
@Override
public void setProperties(Properties p) {
log.debug("Set properties " + host);
}
@Override
public void exit() {
log.debug("Exitting " + host);
}
@Override
public boolean isActive() {
log.debug("Check if active " + host);
return false;
}
public void setHost(String host) {
this.host = host;
}
}
}

View File

@ -56,6 +56,7 @@ Summary
<p>
<ul>
<li><bugzilla>48799</bugzilla> - Add time to establish connection to available sample metrics. Implemented by Andrey Pokhilko (andrey at blazemeter.com) and contributed by BlazeMeter Ltd.</li>
<li><bugzilla>57500</bugzilla> - Introduce retry behavior for distributed testing. Implemented by Andrey Pokhilko and Dzimitry Kashlach and contributed by BlazeMeter Ltd.</li>
<li>Sample text</li>
</ul>
</p>
@ -243,6 +244,7 @@ See <bugzilla>56357</bugzilla> for details.
<li><a href="http://ubikloadpack.com">Ubik Load Pack</a></li>
<li>Yngvi &amp;THORN;&amp;oacute;r Sigurj&amp;oacute;nsson (blitzkopf at gmail.com)</li>
<li>Dzmitry Kashlach (dzmitrykashlach at gmail.com)</li>
<li><a href="http://blazemeter.com">BlazeMeter Ltd.</a></li>
<li>Benoit Wiart (benoit.wiart at gmail.com)</li>
</ul>

View File

@ -282,7 +282,24 @@ There are some JMeter properties that can be set to alter this behaviour.
</subsection>
<subsection name="&sect-num;.5 Dealing with nodes that failed starting" anchor="retries">
<p>
For a large-scale tests there is a chance that some part of remote servers will be unavailable or down.
For example, when you use automation script to allocate many cloud machines and use them as generators,
some of requested machines might fail booting because of cloud's issues.
</p>
<p>
First what you might want is to retry initialization attempts in hope that failed nodes just slightly delayed their boot.
To enable retries, you should set <code>client.tries</code> property to total number of connection attempts.
By default it does only one attempt. To control retry delay, set the <code>client.retries_delay</code> property
to number of milliseconds to sleep between attempts.
</p>
<p>
Finally, you might still want to run the test with those generators that succeeded initialization and skipping failed nodes.
To enable that, set the <code>client.continue_on_fail=true</code> property.
</p>
</subsection>
</section>