You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
{{ message }}
This repository has been archived by the owner on May 28, 2018. It is now read-only.
Using jersey client with apache connector from multiple threads concurrently results in threads that wait forever
How to reproduce
Run the following code
import static javax.ws.rs.HttpMethod.GET;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE;
import java.net.URI;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;
import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.glassfish.jersey.apache.connector.ApacheClientProperties;
import org.glassfish.jersey.apache.connector.ApacheConnectorProvider;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.jackson.JacksonFeature;
import com.spotify.docker.client.LogsResponseReader;
import com.spotify.docker.client.ObjectMapperProvider;
import com.spotify.docker.client.ProgressResponseReader;
import com.spotify.docker.client.UnixConnectionSocketFactory;
import com.spotify.docker.client.messages.Container;
import com.spotify.docker.client.messages.ContainerInfo;
public class Jersey {
public static void main(String[] args) throws InterruptedException {
final URI originalUri = URI.create("unix:///var/run/docker.sock");
final URI uri = UnixConnectionSocketFactory.sanitizeUri(originalUri);
final Registry<ConnectionSocketFactory> registry = RegistryBuilder
.<ConnectionSocketFactory>create()
.register("http", PlainConnectionSocketFactory.getSocketFactory())
.register("unix", new UnixConnectionSocketFactory(originalUri))
.build();
final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
cm.setMaxTotal(100);
cm.setDefaultMaxPerRoute(100);
final RequestConfig requestConfig = RequestConfig.custom()
.setConnectionRequestTimeout(30000)
.setConnectTimeout(30000)
.setSocketTimeout(5000)
.build();
final ClientConfig config = new ClientConfig(
ObjectMapperProvider.class,
JacksonFeature.class,
LogsResponseReader.class,
ProgressResponseReader.class)
.connectorProvider(new ApacheConnectorProvider())
.property(ApacheClientProperties.CONNECTION_MANAGER, cm)
.property(ApacheClientProperties.REQUEST_CONFIG, requestConfig);
final Client client = ClientBuilder.newBuilder()
.withConfig(config)
.build();
final int threadCount = 100;
final CountDownLatch startLatch = new CountDownLatch(threadCount);
final CountDownLatch endLatch = new CountDownLatch(threadCount);
for (int i = 0; i < threadCount; ++i) {
new Thread(new Worker(i, client, uri, startLatch, endLatch)).start();
}
endLatch.await();
System.out.println("All done");
client.close();
}
static class Worker implements Runnable {
private final Client client;
private final CountDownLatch myEndLatch;
private final URI uri;
private final CountDownLatch myStartLatch;
private final int myI;
private static final GenericType<List<Container>> CONTAINER_LIST =
new GenericType<List<Container>>() {
};
Worker(final int i,
final Client client,
final URI uri,
final CountDownLatch startLatch,
final CountDownLatch endLatch) {
myI = i;
this.client = client;
this.uri = uri;
myStartLatch = startLatch;
myEndLatch = endLatch;
}
public void run() {
try {
myStartLatch.countDown();
myStartLatch.await();
for (int i = 0; i < 10; i++) {
List<Container> containers = listContainers();
for (Container container : containers) {
// These two methods do the same thing - the first one uses ContainerInfo as result
// type, while the second one uses String as result type. When using the first one the threads
// will get stuck, while with the second one program works as expected
System.out.println(inspectContainer(container.id()));
// inspectContainerReturnString(container.id());
}
ping();
}
} catch (final Throwable e) {
e.printStackTrace();
} finally {
System.out.println("done " + myI);
myEndLatch.countDown();
}
}
List<Container> listContainers() throws InterruptedException {
final WebTarget resource = client.target(uri).path("v1.27").path("containers").path("json").queryParam("all", 1);
return request(GET, CONTAINER_LIST, resource.request(APPLICATION_JSON_TYPE));
}
ContainerInfo inspectContainer(String containerId) throws InterruptedException {
final WebTarget resource = client.target(uri).path("v1.27").path("containers").path(containerId).path("json");
return request(GET, ContainerInfo.class, resource.request(APPLICATION_JSON_TYPE));
}
String inspectContainerReturnString(String containerId) throws InterruptedException {
final WebTarget resource = client.target(uri).path("v1.27").path("containers").path(containerId).path("json");
return request(GET, String.class, resource.request(APPLICATION_JSON_TYPE));
}
String ping() throws InterruptedException {
final WebTarget resource = client.target(uri).path("v1.27").path("_ping");
return request(GET, String.class, resource.request(APPLICATION_JSON_TYPE));
}
private <T> T request(final String method, final Class<T> clazz, final Invocation.Builder request)
throws InterruptedException {
try {
return request.async().method(method, clazz).get();
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
private <T> T request(final String method, final GenericType<T> type, final Invocation.Builder request)
throws InterruptedException {
try {
return request.async().method(method, type).get();
} catch (ExecutionException e) {
throw new RuntimeException(e);
}
}
}
}
"All done" is written in the console and (optionally, see notes below) the program finishes
Actual result
One of the threads gets stuck with the following thread dump output
"Thread-1" #11 prio=5 os_prio=31 tid=0x00007f992a19b000 nid=0x3f03 waiting on condition [0x00007000023fc000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000786f7cda0> (a java.util.concurrent.CompletableFuture$Signaller)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1693)
at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323)
at java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1729)
at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895)
at Jersey$Worker.request(Jersey.java:152)
at Jersey$Worker.inspectContainer(Jersey.java:136)
at Jersey$Worker.run(Jersey.java:115)
at java.lang.Thread.run(Thread.java:748)
Prerequisites
docker (any version)
A few containers shoud be present (can be in stopped state)
docker run busybox /bin/true
docker run busybox /bin/true
docker run busybox /bin/true
Notes
There should be at least a few docker containers present on the machine
Replacing inspectContainer call with inspectContainerReturnString will make the program run as expected, without threads getting stuck
I managed to reproduce this with ApacheConnector, communicating over UNIX socket with docker daemon (this is how communication is performed with docker daemon in general)
I couldn't reproduce this with regular http calls, i.e. by using final Client client = ClientBuilder.newBuilder().build(); and communicating with a dummy http server that returns over http the same responses that daemon would return over UNIX socket
This can be reproduced with jersey versions >= 2.22.2
Jersey 2.22.1 does not cause this issue, although it suffers from JERSEY-2967 which prevents the program from finishing, but it outputs "All done" in the test case and there is no deadlock mentioned above
Changes in jersey/jersey@b5b9fb6could be relevant, but it's just a guess, maybe it's not related to apache connector at all
The above thread dump looks slightly different when using other jersey versions, e.g. with 2.22.2 it looks like
"Thread-93" #103 prio=5 os_prio=31 tid=0x00007f9387a36800 nid=0xd203 waiting on condition [0x000070000aba2000]
java.lang.Thread.State: WAITING (parking)
at sun.misc.Unsafe.park(Native Method)
- parking to wait for <0x0000000788278e88> (a jersey.repackaged.com.google.common.util.concurrent.AbstractFuture$Sync)
at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
at jersey.repackaged.com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:285)
at jersey.repackaged.com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
at Jersey$Worker.request(Jersey.java:152)
at Jersey$Worker.inspectContainer(Jersey.java:136)
at Jersey$Worker.run(Jersey.java:115)
at java.lang.Thread.run(Thread.java:748)
The text was updated successfully, but these errors were encountered:
Description
Using jersey client with apache connector from multiple threads concurrently results in threads that wait forever
How to reproduce
Run the following code
With this maven pom.xml
Expected result
"All done" is written in the console and (optionally, see notes below) the program finishes
Actual result
One of the threads gets stuck with the following thread dump output
Prerequisites
Notes
inspectContainer
call withinspectContainerReturnString
will make the program run as expected, without threads getting stuckfinal Client client = ClientBuilder.newBuilder().build();
and communicating with a dummy http server that returns over http the same responses that daemon would return over UNIX socketThe text was updated successfully, but these errors were encountered: