8197589: Update CPU count algorithm when both cpu shares and quotas are used

Reviewed-by: dholmes, mseledtsov

src/hotspot/os/linux/globals_linux.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2016, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -62,6 +62,11 @@
   product(bool, UseContainerSupport, true,                              \
           "Enable detection and runtime container configuration support") \
                                                                         \
+  product(bool, PreferContainerQuotaForCPUCount, true,                  \
+          "Calculate the container CPU availability based on the value" \
+          " of quotas (if set), when true. Otherwise, use the CPU"    \
+          " shares value, provided it is less than quota.")             \
+                                                                        \
   diagnostic(bool, UseCpuAllocPath, false,                              \
              "Use CPU_ALLOC code path in os::active_processor_count ")
 

src/hotspot/os/linux/osContainer_linux.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -499,11 +499,11 @@ jlong OSContainer::memory_max_usage_in_bytes() {
 /* active_processor_count
  *
  * Calculate an appropriate number of active processors for the
- * VM to use based on these three cgroup options.
+ * VM to use based on these three inputs.
  *
  * cpu affinity
- * cpu quota & cpu period
+ * cgroup cpu quota & cpu period
- * cpu shares
+ * cgroup cpu shares
  *
  * Algorithm:
  *
@@ -513,42 +513,61 @@ jlong OSContainer::memory_max_usage_in_bytes() {
  * required CPUs by dividing quota by period.
  *
  * If shares are in effect (shares != -1), calculate the number
- * of cpus required for the shares by dividing the share value
+ * of CPUs required for the shares by dividing the share value
  * by PER_CPU_SHARES.
  *
  * All results of division are rounded up to the next whole number.
  *
- * Return the smaller number from the three different settings.
+ * If neither shares or quotas have been specified, return the
+ * number of active processors in the system.
+ *
+ * If both shares and quotas have been specified, the results are
+ * based on the flag PreferContainerQuotaForCPUCount.  If true,
+ * return the quota value.  If false return the smallest value
+ * between shares or quotas.
+ *
+ * If shares and/or quotas have been specified, the resulting number
+ * returned will never exceed the number of active processors.
  *
  * return:
- *    number of cpus
+ *    number of CPUs
- *    OSCONTAINER_ERROR if failure occured during extract of cpuset info
  */
 int OSContainer::active_processor_count() {
-  int cpu_count, share_count, quota_count;
+  int quota_count = 0, share_count = 0;
-  int share, quota, period;
+  int cpu_count, limit_count;
   int result;
 
-  cpu_count = os::Linux::active_processor_count();
+  cpu_count = limit_count = os::Linux::active_processor_count();
+  int quota  = cpu_quota();
+  int period = cpu_period();
+  int share  = cpu_shares();
 
-  share = cpu_shares();
-  if (share > -1) {
-    share_count = ceilf((float)share / (float)PER_CPU_SHARES);
-    log_trace(os, container)("cpu_share count: %d", share_count);
-  } else {
-    share_count = cpu_count;
-  }
-
-  quota = cpu_quota();
-  period = cpu_period();
   if (quota > -1 && period > 0) {
     quota_count = ceilf((float)quota / (float)period);
-    log_trace(os, container)("quota_count: %d", quota_count);
+    log_trace(os, container)("CPU Quota count based on quota/period: %d", quota_count);
-  } else {
+  }
-    quota_count = cpu_count;
+  if (share > -1) {
+    share_count = ceilf((float)share / (float)PER_CPU_SHARES);
+    log_trace(os, container)("CPU Share count based on shares: %d", share_count);
   }
 
-  result = MIN2(cpu_count, MIN2(share_count, quota_count));
+  // If both shares and quotas are setup results depend
+  // on flag PreferContainerQuotaForCPUCount.
+  // If true, limit CPU count to quota
+  // If false, use minimum of shares and quotas
+  if (quota_count !=0 && share_count != 0) {
+    if (PreferContainerQuotaForCPUCount) {
+      limit_count = quota_count;
+    } else {
+      limit_count = MIN2(quota_count, share_count);
+    }
+  } else if (quota_count != 0) {
+    limit_count = quota_count;
+  } else if (share_count != 0) {
+    limit_count = share_count;
+  }
+
+  result = MIN2(cpu_count, limit_count);
   log_trace(os, container)("OSContainer::active_processor_count: %d", result);
   return result;
 }

test/hotspot/jtreg/runtime/containers/docker/TestCPUAwareness.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2017, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2017, 2018, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -94,14 +94,23 @@ public class TestCPUAwareness {
             // Test subset of cpuset with one element
             if (cpuSet.size() >= 1) {
                 String testCpuSet = CPUSetsReader.listToString(cpuSet, 1);
-                testAPCCombo(testCpuSet, 200*1000, 100*1000,   4*1024, 1);
+                testAPCCombo(testCpuSet, 200*1000, 100*1000,   4*1024, true, 1);
             }
 
             // Test subset of cpuset with two elements
             if (cpuSet.size() >= 2) {
                 String testCpuSet = CPUSetsReader.listToString(cpuSet, 2);
-                testAPCCombo(testCpuSet, 200*1000, 100*1000, 4*1024, 2);
+                testAPCCombo(testCpuSet, 200*1000, 100*1000, 4*1024, true, 2);
-                testAPCCombo(testCpuSet, 200*1000, 100*1000, 1*1024, 2);
+                testAPCCombo(testCpuSet, 200*1000, 100*1000, 1023,   true, 2);
+                testAPCCombo(testCpuSet, 200*1000, 100*1000, 1023,   false,  1);
+            }
+
+            // Test subset of cpuset with three elements
+            if (cpuSet.size() >= 3) {
+                String testCpuSet = CPUSetsReader.listToString(cpuSet, 3);
+                testAPCCombo(testCpuSet, 100*1000, 100*1000, 2*1024, true, 1);
+                testAPCCombo(testCpuSet, 200*1000, 100*1000, 1023,   true, 2);
+                testAPCCombo(testCpuSet, 200*1000, 100*1000, 1023,   false,  1);
             }
         }
     }
@@ -159,12 +168,14 @@ public class TestCPUAwareness {
 
     // Test correctess of automatically selected active processor cound
     private static void testAPCCombo(String cpuset, int quota, int period, int shares,
-                                         int expectedAPC) throws Exception {
+                                     boolean usePreferContainerQuotaForCPUCount,
+                                     int expectedAPC) throws Exception {
         Common.logNewTestCase("test APC Combo");
         System.out.println("cpuset = " + cpuset);
         System.out.println("quota = " + quota);
         System.out.println("period = " + period);
         System.out.println("shares = " + period);
+        System.out.println("usePreferContainerQuotaForCPUCount = " + usePreferContainerQuotaForCPUCount);
         System.out.println("expectedAPC = " + expectedAPC);
 
         expectedAPC = adjustExpectedAPCForAvailableCPUs(expectedAPC);
@@ -174,6 +185,9 @@ public class TestCPUAwareness {
             .addDockerOpts("--cpu-period=" + period)
             .addDockerOpts("--cpu-quota=" + quota)
             .addDockerOpts("--cpu-shares=" + shares);
+
+        if (!usePreferContainerQuotaForCPUCount) opts.addJavaOpts("-XX:-PreferContainerQuotaForCPUCount");
+
         Common.run(opts)
             .shouldMatch("active_processor_count.*" + expectedAPC);
     }