Threads stuck when jersey client with apache connector is used concurrently

[daniloradenovic]

Description

Using jersey client with apache connector from multiple threads concurrently results in threads that wait forever

How to reproduce

Run the following code

import static javax.ws.rs.HttpMethod.GET;
import static javax.ws.rs.core.MediaType.APPLICATION_JSON_TYPE;

import java.net.URI;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutionException;

import javax.ws.rs.client.Client;
import javax.ws.rs.client.ClientBuilder;
import javax.ws.rs.client.Invocation;
import javax.ws.rs.client.WebTarget;
import javax.ws.rs.core.GenericType;

import org.apache.http.client.config.RequestConfig;
import org.apache.http.config.Registry;
import org.apache.http.config.RegistryBuilder;
import org.apache.http.conn.socket.ConnectionSocketFactory;
import org.apache.http.conn.socket.PlainConnectionSocketFactory;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.glassfish.jersey.apache.connector.ApacheClientProperties;
import org.glassfish.jersey.apache.connector.ApacheConnectorProvider;
import org.glassfish.jersey.client.ClientConfig;
import org.glassfish.jersey.jackson.JacksonFeature;

import com.spotify.docker.client.LogsResponseReader;
import com.spotify.docker.client.ObjectMapperProvider;
import com.spotify.docker.client.ProgressResponseReader;
import com.spotify.docker.client.UnixConnectionSocketFactory;
import com.spotify.docker.client.messages.Container;
import com.spotify.docker.client.messages.ContainerInfo;

public class Jersey {

  public static void main(String[] args) throws InterruptedException {

    final URI originalUri = URI.create("unix:///var/run/docker.sock");
    final URI uri = UnixConnectionSocketFactory.sanitizeUri(originalUri);

    final Registry<ConnectionSocketFactory> registry = RegistryBuilder
        .<ConnectionSocketFactory>create()
        .register("http", PlainConnectionSocketFactory.getSocketFactory())
        .register("unix", new UnixConnectionSocketFactory(originalUri))
        .build();

    final PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(registry);
    cm.setMaxTotal(100);
    cm.setDefaultMaxPerRoute(100);

    final RequestConfig requestConfig = RequestConfig.custom()
        .setConnectionRequestTimeout(30000)
        .setConnectTimeout(30000)
        .setSocketTimeout(5000)
        .build();

    final ClientConfig config = new ClientConfig(
        ObjectMapperProvider.class,
        JacksonFeature.class,
        LogsResponseReader.class,
        ProgressResponseReader.class)
        .connectorProvider(new ApacheConnectorProvider())
        .property(ApacheClientProperties.CONNECTION_MANAGER, cm)
        .property(ApacheClientProperties.REQUEST_CONFIG, requestConfig);

    final Client client = ClientBuilder.newBuilder()
        .withConfig(config)
        .build();

    final int threadCount = 100;

    final CountDownLatch startLatch = new CountDownLatch(threadCount);
    final CountDownLatch endLatch = new CountDownLatch(threadCount);
    for (int i = 0; i < threadCount; ++i) {
      new Thread(new Worker(i, client, uri, startLatch, endLatch)).start();
    }
    endLatch.await();
    System.out.println("All done");

    client.close();
  }

  static class Worker implements Runnable {
    private final Client client;
    private final CountDownLatch myEndLatch;
    private final URI uri;
    private final CountDownLatch myStartLatch;
    private final int myI;
    private static final GenericType<List<Container>> CONTAINER_LIST =
        new GenericType<List<Container>>() {
        };

    Worker(final int i,
           final Client client,
           final URI uri,
           final CountDownLatch startLatch,
           final CountDownLatch endLatch) {
      myI = i;
      this.client = client;
      this.uri = uri;
      myStartLatch = startLatch;
      myEndLatch = endLatch;
    }

    public void run() {

      try {
        myStartLatch.countDown();
        myStartLatch.await();
        for (int i = 0; i < 10; i++) {
          List<Container> containers = listContainers();
          for (Container container : containers) {
            // These two methods do the same thing - the first one uses ContainerInfo as result
            // type, while the second one uses String as result type. When using the first one the threads
            // will get stuck, while with the second one program works as expected
            System.out.println(inspectContainer(container.id()));
//            inspectContainerReturnString(container.id());
          }
          ping();
        }
      } catch (final Throwable e) {
        e.printStackTrace();
      } finally {
        System.out.println("done " + myI);
        myEndLatch.countDown();
      }
    }

    List<Container> listContainers() throws InterruptedException {
      final WebTarget resource = client.target(uri).path("v1.27").path("containers").path("json").queryParam("all", 1);

      return request(GET, CONTAINER_LIST, resource.request(APPLICATION_JSON_TYPE));
    }

    ContainerInfo inspectContainer(String containerId) throws InterruptedException {
      final WebTarget resource = client.target(uri).path("v1.27").path("containers").path(containerId).path("json");
      return request(GET, ContainerInfo.class, resource.request(APPLICATION_JSON_TYPE));
    }

    String inspectContainerReturnString(String containerId) throws InterruptedException {
      final WebTarget resource = client.target(uri).path("v1.27").path("containers").path(containerId).path("json");
      return request(GET, String.class, resource.request(APPLICATION_JSON_TYPE));
    }

    String ping() throws InterruptedException {
      final WebTarget resource = client.target(uri).path("v1.27").path("_ping");
      return request(GET, String.class, resource.request(APPLICATION_JSON_TYPE));
    }

    private <T> T request(final String method, final Class<T> clazz, final Invocation.Builder request)
        throws InterruptedException {
      try {
        return request.async().method(method, clazz).get();
      } catch (ExecutionException e) {
        throw new RuntimeException(e);
      }
    }

    private <T> T request(final String method, final GenericType<T> type, final Invocation.Builder request)
        throws InterruptedException {
      try {
        return request.async().method(method, type).get();
      } catch (ExecutionException e) {
        throw new RuntimeException(e);
      }
    }
  }
}

With this maven pom.xml

<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0"
         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  <modelVersion>4.0.0</modelVersion>

  <groupId>docker-client-debug</groupId>
  <artifactId>docker-client-debug</artifactId>
  <version>1.0-SNAPSHOT</version>

  <properties>
    <jersey.version>2.26</jersey.version>
  </properties>
  <build>
    <plugins>
      <plugin>
        <groupId>org.apache.maven.plugins</groupId>
        <artifactId>maven-compiler-plugin</artifactId>
        <configuration>
          <source>1.8</source>
          <target>1.8</target>
        </configuration>
      </plugin>
    </plugins>
  </build>

  <dependencies>
    <dependency>
      <groupId>com.spotify</groupId>
      <artifactId>docker-client</artifactId>
      <version>8.11.1</version>
    </dependency>
    <dependency>
      <groupId>org.glassfish.jersey.core</groupId>
      <artifactId>jersey-client</artifactId>
      <version>${jersey.version}</version>
    </dependency>
    <dependency>
      <groupId>org.glassfish.jersey.containers</groupId>
      <artifactId>jersey-container-servlet</artifactId>
      <version>${jersey.version}</version>
    </dependency>
    <dependency>
      <groupId>org.glassfish.jersey.connectors</groupId>
      <artifactId>jersey-apache-connector</artifactId>
      <version>${jersey.version}</version>
    </dependency>
    <dependency>
      <groupId>org.glassfish.jersey.inject</groupId>
      <artifactId>jersey-hk2</artifactId>
      <version>${jersey.version}</version>
    </dependency>
  </dependencies>
</project>

Expected result

"All done" is written in the console and (optionally, see notes below) the program finishes

Actual result

One of the threads gets stuck with the following thread dump output

"Thread-1" #11 prio=5 os_prio=31 tid=0x00007f992a19b000 nid=0x3f03 waiting on condition [0x00007000023fc000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x0000000786f7cda0> (a java.util.concurrent.CompletableFuture$Signaller)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1693)
	at java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3323)
	at java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1729)
	at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:1895)
	at Jersey$Worker.request(Jersey.java:152)
	at Jersey$Worker.inspectContainer(Jersey.java:136)
	at Jersey$Worker.run(Jersey.java:115)
	at java.lang.Thread.run(Thread.java:748)

Prerequisites

• docker (any version)
• A few containers shoud be present (can be in stopped state)

docker run busybox /bin/true
docker run busybox /bin/true
docker run busybox /bin/true

Notes

• There should be at least a few docker containers present on the machine
• Replacing inspectContainer call with inspectContainerReturnString will make the program run as expected, without threads getting stuck
• I managed to reproduce this with ApacheConnector, communicating over UNIX socket with docker daemon (this is how communication is performed with docker daemon in general)
• I couldn't reproduce this with regular http calls, i.e. by using final Client client = ClientBuilder.newBuilder().build(); and communicating with a dummy http server that returns over http the same responses that daemon would return over UNIX socket
• This can be reproduced with jersey versions >= 2.22.2
• Jersey 2.22.1 does not cause this issue, although it suffers from JERSEY-2967 which prevents the program from finishing, but it outputs "All done" in the test case and there is no deadlock mentioned above
• Changes in jersey/jersey@b5b9fb6 could be relevant, but it's just a guess, maybe it's not related to apache connector at all
• Maybe this is the same as Park or deadlock thread on callSendError. #3558
• The above thread dump looks slightly different when using other jersey versions, e.g. with 2.22.2 it looks like

"Thread-93" #103 prio=5 os_prio=31 tid=0x00007f9387a36800 nid=0xd203 waiting on condition [0x000070000aba2000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x0000000788278e88> (a jersey.repackaged.com.google.common.util.concurrent.AbstractFuture$Sync)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.parkAndCheckInterrupt(AbstractQueuedSynchronizer.java:836)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.doAcquireSharedInterruptibly(AbstractQueuedSynchronizer.java:997)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1304)
	at jersey.repackaged.com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:285)
	at jersey.repackaged.com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
	at Jersey$Worker.request(Jersey.java:152)
	at Jersey$Worker.inspectContainer(Jersey.java:136)
	at Jersey$Worker.run(Jersey.java:115)
	at java.lang.Thread.run(Thread.java:748)