[PATCH]: Limit number of compiler instances in JavacServer to avoid OOM

[Per Lidén]

Hi all,

When I'm building on a 16 core machine the JavacServer will currently 
OOM. On this machine the ./configure script will correctly figure out 
that we should allow a maximum of 7 instances with the given 3G heap, 
and sets JAVAC_SERVER_CORES accordingly. However, JavacServer later 
fails to properly limit the number of compiler instances to 
JAVAC_SERVER_CORES, instead allowing a new compiler instances to be 
created for each concurrent connection it accepts (which will be equal 
to NUM_CORES, i.e. 16 in this case), resulting in an OOM. The 
JAVAC_SERVER_CORES value currently only dictates how many compilers can 
run concurrently, not the number of instances it's allowed to create.

The patch fixes this by making sure Handler.grabHandle() never grows the 
handler pool to more than JAVAC_SERVER_CORES (which propagates into 
JavacServer as "poolsize").

Further, since the thread accepting incomming connections now risk 
blocking for a handler to become available, we need to increase the 
backlog on the server socket to avoid connection failures in the 
clients. This number should probably be something like NUM_CORE (or we 
should spawn a thread for each connection we accept), but right now I 
just increased it to 128. Keeping it at default 0 will not work.


diff --git 
a/src/share/classes/com/sun/tools/javacserver/JavacServer.java 
b/src/share/classes/com/sun/tools/javacserver/JavacServer.java
--- a/src/share/classes/com/sun/tools/javacserver/JavacServer.java
+++ b/src/share/classes/com/sun/tools/javacserver/JavacServer.java
@@ -53,10 +53,12 @@
  import java.util.ArrayList;
  import java.util.Arrays;
  import java.util.List;
+import java.util.Stack;
  import java.util.Random;
  import java.util.concurrent.ExecutorService;
  import java.util.concurrent.Executors;
  import java.util.concurrent.TimeUnit;
+import java.util.concurrent.Semaphore;
  import java.util.logging.Logger;

  import javax.tools.ForwardingJavaFileManager;
@@ -286,9 +288,10 @@
          server_start = System.currentTimeMillis();
          // Create a server socket on a random port that is bound to 
the localhost/127.0.0.1 interface.
          // I.e only local processes can connect to this port.
-        serverSocket = new ServerSocket(0,0,InetAddress.getByName(null));
+        serverSocket = new ServerSocket(0,128,InetAddress.getByName(null));
          int port = serverSocket.getLocalPort();
          pool = Executors.newFixedThreadPool(poolSize);
+        Handler.setPoolSize(poolSize);
          Random rnd = new Random();
          my_cookie = rnd.nextLong();
          si.setValues(port, my_cookie);
@@ -555,29 +558,44 @@
      }

      static class Handler implements Runnable {
-        private static Handler[] handlers = new Handler[0];
-        private static int available_handlers = 0;
+        private static Semaphore available;
+        private static Stack<Handler> handlers = new Stack<Handler>();

-        public static synchronized Handler grabHandler() {
-            if (available_handlers==0) {
-                handlers = new Handler[handlers.length+1];
+        public static void setPoolSize(int poolsize) {
+            if (available != null) {
+                log("Internal Error: Handler pool size already set!");
+                System.exit(-1);
+            }
+            available = new Semaphore(poolsize, true);
+        }
+
+        public static Handler grabHandler() throws InterruptedException {
+            available.acquire();
+            return popHandler();
+        }
+
+        private static synchronized Handler popHandler() {
+            if (handlers.empty()) {
                  return new Handler();
              }
-            available_handlers--;
-            Handler tmp = handlers[available_handlers];
-            tmp.returned = false;
-            handlers[available_handlers] = null;
-            return tmp;
+
+            Handler h = handlers.pop();
+           h.returned = false;
+           return h;
          }

-        public static synchronized void returnHandler(Handler h) {
-            if (h.returned == true) {
+        public static void returnHandler(Handler h) {
+            pushHandler(h);
+            available.release();
+        }
+
+        private static synchronized void pushHandler(Handler h) {
+            if (h.returned) {
                                 log("Internal Error: Handler already 
returned once!");
                  System.exit(-1);
              }
              h.returned = true;
-            handlers[available_handlers] = h;
-            available_handlers++;
+            handlers.push(h);
          }

          private boolean returned = false;


/Per