8004352: build-infra: Limit JOBS on large machinesjdk8-b80

Reviewed-by: mduigou

8 files changed, 102 insertions, 40 deletions

diff --git a/common/autoconf/build-performance.m4 b/common/autoconf/build-performance.m4
index 0d1d8b9..1207541 100644
--- a/common/autoconf/build-performance.m4
+++ b/common/autoconf/build-performance.m4
@@ -47,10 +47,6 @@ AC_DEFUN([BPERF_CHECK_CORES],
         FOUND_CORES=yes
     fi
 
-    # For c/c++ code we run twice as many concurrent build
-    # jobs than we have cores, otherwise we will stall on io.
-    CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2`
-
     if test "x$FOUND_CORES" = xyes; then
         AC_MSG_RESULT([$NUM_CORES])
     else
@@ -98,32 +94,62 @@ AC_DEFUN([BPERF_CHECK_MEMORY_SIZE],
 
 AC_DEFUN_ONCE([BPERF_SETUP_BUILD_CORES],
 [
-# How many cores do we have on this build system?
-AC_ARG_WITH(num-cores, [AS_HELP_STRING([--with-num-cores],
+  # How many cores do we have on this build system?
+  AC_ARG_WITH(num-cores, [AS_HELP_STRING([--with-num-cores],
     [number of cores in the build system, e.g. --with-num-cores=8 @<:@probed@:>@])])
-if test "x$with_num_cores" = x; then
+  if test "x$with_num_cores" = x; then
     # The number of cores were not specified, try to probe them.
     BPERF_CHECK_CORES
-else
+  else
     NUM_CORES=$with_num_cores
-    CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2`
-fi
-AC_SUBST(NUM_CORES)
-AC_SUBST(CONCURRENT_BUILD_JOBS)
+  fi
+  AC_SUBST(NUM_CORES)
 ])
 
 AC_DEFUN_ONCE([BPERF_SETUP_BUILD_MEMORY],
 [
-# How much memory do we have on this build system?
-AC_ARG_WITH(memory-size, [AS_HELP_STRING([--with-memory-size],
+  # How much memory do we have on this build system?
+  AC_ARG_WITH(memory-size, [AS_HELP_STRING([--with-memory-size],
     [memory (in MB) available in the build system, e.g. --with-memory-size=1024 @<:@probed@:>@])])
-if test "x$with_memory_size" = x; then
+  if test "x$with_memory_size" = x; then
     # The memory size was not specified, try to probe it.
     BPERF_CHECK_MEMORY_SIZE
-else
+  else
     MEMORY_SIZE=$with_memory_size
-fi
-AC_SUBST(MEMORY_SIZE)
+  fi
+  AC_SUBST(MEMORY_SIZE)
+])
+
+AC_DEFUN_ONCE([BPERF_SETUP_BUILD_JOBS],
+[
+  # Provide a decent default number of parallel jobs for make depending on 
+  # number of cores, amount of memory and machine architecture.
+  AC_ARG_WITH(jobs, [AS_HELP_STRING([--with-jobs],
+    [number of parallel jobs to let make run @<:@calculated based on cores and memory@:>@])])
+  if test "x$with_jobs" = x; then
+    # Number of jobs was not specified, calculate.
+    AC_MSG_CHECKING([for appropriate number of jobs to run in parallel])
+    # Approximate memory in GB, rounding up a bit.
+    memory_gb=`expr $MEMORY_SIZE / 1100`
+    # Pick the lowest of memory in gb and number of cores.
+    if test "$memory_gb" -lt "$NUM_CORES"; then
+      JOBS="$memory_gb"
+    else
+      JOBS="$NUM_CORES"
+      # On bigger machines, leave some room for other processes to run
+      if test "$JOBS" -gt "4"; then
+        JOBS=`expr $JOBS '*' 90 / 100`
+      fi
+    fi
+    # Cap number of jobs to 16
+    if test "$JOBS" -gt "16"; then
+      JOBS=16
+    fi
+    AC_MSG_RESULT([$JOBS])
+  else
+    JOBS=$with_jobs
+  fi
+  AC_SUBST(JOBS)
 ])
 
 AC_DEFUN([BPERF_SETUP_CCACHE],
diff --git a/common/autoconf/configure.ac b/common/autoconf/configure.ac
index 0bb0faf..da9996f 100644
--- a/common/autoconf/configure.ac
+++ b/common/autoconf/configure.ac
@@ -204,6 +204,7 @@ JDKOPT_SETUP_BUILD_TWEAKS
 
 BPERF_SETUP_BUILD_CORES
 BPERF_SETUP_BUILD_MEMORY
+BPERF_SETUP_BUILD_JOBS
 
 # Setup smart javac (after cores and memory have been setup)
 BPERF_SETUP_SMART_JAVAC
diff --git a/common/autoconf/generated-configure.sh b/common/autoconf/generated-configure.sh
index 3814bea..31764ab 100644
--- a/common/autoconf/generated-configure.sh
+++ b/common/autoconf/generated-configure.sh
@@ -601,8 +601,8 @@ SJAVAC_SERVER_DIR
 ENABLE_SJAVAC
 SJAVAC_SERVER_CORES
 SJAVAC_SERVER_JAVA
+JOBS
 MEMORY_SIZE
-CONCURRENT_BUILD_JOBS
 NUM_CORES
 SALIB_NAME
 HOTSPOT_MAKE_ARGS
@@ -1002,6 +1002,7 @@ with_zlib
 with_stdc__lib
 with_num_cores
 with_memory_size
+with_jobs
 with_sjavac_server_java
 with_sjavac_server_cores
 enable_sjavac
@@ -1760,6 +1761,8 @@ Optional Packages:
                           --with-num-cores=8 [probed]
   --with-memory-size      memory (in MB) available in the build system, e.g.
                           --with-memory-size=1024 [probed]
+  --with-jobs             number of parallel jobs to let make run [calculated
+                          based on cores and memory]
   --with-sjavac-server-java
                           use this java binary for running the sjavac
                           background server [Boot JDK java]
@@ -3329,6 +3332,8 @@ ac_configure="$SHELL $ac_aux_dir/configure"  # Please don't use this var.
 
 
 
+
+
 #
 # Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
@@ -3724,7 +3729,7 @@ fi
 #CUSTOM_AUTOCONF_INCLUDE
 
 # Do not change or remove the following line, it is needed for consistency checks:
-DATE_WHEN_GENERATED=1361452590
+DATE_WHEN_GENERATED=1362411827
 
 ###############################################################################
 #
@@ -31225,14 +31230,14 @@ fi
 ###############################################################################
 
 
-# How many cores do we have on this build system?
+  # How many cores do we have on this build system?
 
 # Check whether --with-num-cores was given.
 if test "${with_num_cores+set}" = set; then :
   withval=$with_num_cores;
 fi
 
-if test "x$with_num_cores" = x; then
+  if test "x$with_num_cores" = x; then
     # The number of cores were not specified, try to probe them.
 
     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for number of cores" >&5
@@ -31258,10 +31263,6 @@ $as_echo_n "checking for number of cores... " >&6; }
         FOUND_CORES=yes
     fi
 
-    # For c/c++ code we run twice as many concurrent build
-    # jobs than we have cores, otherwise we will stall on io.
-    CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2`
-
     if test "x$FOUND_CORES" = xyes; then
         { $as_echo "$as_me:${as_lineno-$LINENO}: result: $NUM_CORES" >&5
 $as_echo "$NUM_CORES" >&6; }
@@ -31273,22 +31274,20 @@ $as_echo "$as_me: WARNING: This will disable all parallelism from build!" >&2;}
     fi
 
 
-else
+  else
     NUM_CORES=$with_num_cores
-    CONCURRENT_BUILD_JOBS=`expr $NUM_CORES \* 2`
-fi
-
+  fi
 
 
 
-# How much memory do we have on this build system?
+  # How much memory do we have on this build system?
 
 # Check whether --with-memory-size was given.
 if test "${with_memory_size+set}" = set; then :
   withval=$with_memory_size;
 fi
 
-if test "x$with_memory_size" = x; then
+  if test "x$with_memory_size" = x; then
     # The memory size was not specified, try to probe it.
 
     { $as_echo "$as_me:${as_lineno-$LINENO}: checking for memory size" >&5
@@ -31328,10 +31327,46 @@ $as_echo "could not detect memory size, defaulting to 1024 MB" >&6; }
 $as_echo "$as_me: WARNING: This might seriously impact build performance!" >&2;}
     fi
 
-else
+  else
     MEMORY_SIZE=$with_memory_size
+  fi
+
+
+
+  # Provide a decent default number of parallel jobs for make depending on
+  # number of cores, amount of memory and machine architecture.
+
+# Check whether --with-jobs was given.
+if test "${with_jobs+set}" = set; then :
+  withval=$with_jobs;
 fi
 
+  if test "x$with_jobs" = x; then
+    # Number of jobs was not specified, calculate.
+    { $as_echo "$as_me:${as_lineno-$LINENO}: checking for appropriate number of jobs to run in parallel" >&5
+$as_echo_n "checking for appropriate number of jobs to run in parallel... " >&6; }
+    # Approximate memory in GB, rounding up a bit.
+    memory_gb=`expr $MEMORY_SIZE / 1100`
+    # Pick the lowest of memory in gb and number of cores.
+    if test "$memory_gb" -lt "$NUM_CORES"; then
+      JOBS="$memory_gb"
+    else
+      JOBS="$NUM_CORES"
+      # On bigger machines, leave some room for other processes to run
+      if test "$JOBS" -gt "4"; then
+        JOBS=`expr $JOBS '*' 90 / 100`
+      fi
+    fi
+    # Cap number of jobs to 16
+    if test "$JOBS" -gt "16"; then
+      JOBS=16
+    fi
+    { $as_echo "$as_me:${as_lineno-$LINENO}: result: $JOBS" >&5
+$as_echo "$JOBS" >&6; }
+  else
+    JOBS=$with_jobs
+  fi
+
 
 
 # Setup smart javac (after cores and memory have been setup)
@@ -33143,7 +33178,7 @@ printf "* C++ Compiler:   $CXX_VENDOR version $CXX_VERSION (at $CXX)\n"
 
 printf "\n"
 printf "Build performance summary:\n"
-printf "* Cores to use:   $NUM_CORES\n"
+printf "* Cores to use:   $JOBS\n"
 printf "* Memory limit:   $MEMORY_SIZE MB\n"
 printf "* ccache status:  $CCACHE_STATUS\n"
 printf "\n"
diff --git a/common/autoconf/help.m4 b/common/autoconf/help.m4
index 12ff371..cf4a43e 100644
--- a/common/autoconf/help.m4
+++ b/common/autoconf/help.m4
@@ -174,7 +174,7 @@ printf "* C++ Compiler:   $CXX_VENDOR version $CXX_VERSION (at $CXX)\n"
 
 printf "\n"
 printf "Build performance summary:\n"
-printf "* Cores to use:   $NUM_CORES\n"
+printf "* Cores to use:   $JOBS\n"
 printf "* Memory limit:   $MEMORY_SIZE MB\n"
 printf "* ccache status:  $CCACHE_STATUS\n"
 printf "\n"
diff --git a/common/autoconf/hotspot-spec.gmk.in b/common/autoconf/hotspot-spec.gmk.in
index afc887a..5b120e7 100644
--- a/common/autoconf/hotspot-spec.gmk.in
+++ b/common/autoconf/hotspot-spec.gmk.in
@@ -80,7 +80,7 @@ ALT_EXPORT_PATH=$(HOTSPOT_DIST)
 
 HOTSPOT_MAKE_ARGS:=@HOTSPOT_MAKE_ARGS@ @STATIC_CXX_SETTING@
 # This is used from the libjvm build for C/C++ code.
-HOTSPOT_BUILD_JOBS:=@CONCURRENT_BUILD_JOBS@
+HOTSPOT_BUILD_JOBS:=$(JOBS)
 # Control wether Hotspot runs Queens test after building
 TEST_IN_BUILD=@TEST_IN_BUILD@
 
diff --git a/common/autoconf/spec.gmk.in b/common/autoconf/spec.gmk.in
index ef3cacb..358ad78 100644
--- a/common/autoconf/spec.gmk.in
+++ b/common/autoconf/spec.gmk.in
@@ -260,6 +260,9 @@ ENABLE_SJAVAC:=@ENABLE_SJAVAC@
 # the sjavac server log files.
 SJAVAC_SERVER_DIR:=@SJAVAC_SERVER_DIR@
 
+# Number of parallel jobs to use for compilation
+JOBS?=@JOBS@
+
 # The OpenJDK makefiles should be changed to using the standard
 # configure output ..._CFLAGS and ..._LIBS. In the meantime we
 # extract the information here.
diff --git a/common/makefiles/JavaCompilation.gmk b/common/makefiles/JavaCompilation.gmk
index 2725718..bc13da5 100644
--- a/common/makefiles/JavaCompilation.gmk
+++ b/common/makefiles/JavaCompilation.gmk
@@ -501,7 +501,7 @@ define SetupJavaCompilation
 		$(ECHO) Compiling $1
 		($$($1_JVM) $$($1_SJAVAC) \
 			$$($1_REMOTE) \
-			-j $(NUM_CORES) \
+			-j $(JOBS) \
 			--permit-unidentified-artifacts \
 			--permit-sources-without-package \
 			--compare-found-sources $$($1_BIN)/_the.batch.tmp \
diff --git a/common/makefiles/Main.gmk b/common/makefiles/Main.gmk
index 1fce2e2..76a6faa 100644
--- a/common/makefiles/Main.gmk
+++ b/common/makefiles/Main.gmk
@@ -58,9 +58,6 @@ $(eval $(call ResetAllTimers))
 
 # Setup number of jobs to use. -jN is unfortunately not available for us to parse from the command line,
 # hence this workaround.
-ifeq ($(JOBS),)
-    JOBS=$(NUM_CORES)
-endif
 MAKE_ARGS:=$(MAKE_ARGS) -j$(JOBS)
 
 ### Main targets