8039995: Test serviceability/sa/jmap-hashcode/Test8028623.java fails on some Linux/Mac machines

Reviewed-by: dsamersoff, allwin, sla

hotspot/test/testlibrary/com/oracle/java/testlibrary/Platform.java

@@ -23,6 +23,8 @@
 
 package com.oracle.java.testlibrary;
 
+import com.oracle.java.testlibrary.Utils;
+
 public class Platform {
     private static final String osName      = System.getProperty("os.name");
     private static final String dataModel   = System.getProperty("sun.arch.data.model");
@@ -30,6 +32,7 @@ public class Platform {
     private static final String javaVersion = System.getProperty("java.version");
     private static final String osArch      = System.getProperty("os.arch");
     private static final String vmName      = System.getProperty("java.vm.name");
+    private static final String userName    = System.getProperty("user.name");
 
     public static boolean isClient() {
         return vmName.endsWith(" Client VM");
@@ -123,4 +126,56 @@ public class Platform {
         return osArch;
     }
 
+    /**
+     * Return a boolean for whether we expect to be able to attach
+     * the SA to our own processes on this system.
+     */
+    public static boolean shouldSAAttach() throws Exception {
+
+        if (isLinux()) {
+            return canPtraceAttachLinux();
+        } else if (isOSX()) {
+            return canAttachOSX();
+        } else {
+            // Other platforms expected to work:
+            return true;
+        }
+    }
+
+    /**
+     * On Linux, first check the SELinux boolean "deny_ptrace" and return false
+     * as we expect to be denied if that is "1".  Then expect permission to attach
+     * if we are root, so return true.  Then return false for an expected denial
+     * if "ptrace_scope" is 1, and true otherwise.
+     */
+    public static boolean canPtraceAttachLinux() throws Exception {
+
+        // SELinux deny_ptrace:
+        String deny_ptrace = Utils.fileAsString("/sys/fs/selinux/booleans/deny_ptrace");
+        if (deny_ptrace != null && deny_ptrace.contains("1")) {
+            // ptrace will be denied:
+            return false;
+        }
+
+        if (userName.equals("root")) {
+            return true;
+        }
+
+        // ptrace_scope:
+        String ptrace_scope = Utils.fileAsString("/proc/sys/kernel/yama/ptrace_scope");
+        if (ptrace_scope != null && ptrace_scope.contains("1")) {
+            // ptrace will be denied:
+            return false;
+        }
+
+        // Otherwise expect to be permitted:
+        return true;
+    }
+
+    /**
+     * On OSX, expect permission to attach only if we are root.
+     */
+    public static boolean canAttachOSX() throws Exception {
+        return userName.equals("root");
+    }
 }

hotspot/test/testlibrary/com/oracle/java/testlibrary/Utils.java

@@ -313,6 +313,35 @@ public final class Utils {
         return output;
     }
 
+    /**
+     * Return the contents of the named file as a single String,
+     * or null if not found.
+     * @param filename name of the file to read
+     * @return String contents of file, or null if file not found.
+     */
+    public static String fileAsString(String filename) {
+        StringBuilder result = new StringBuilder();
+        try {
+            File file = new File(filename);
+            if (file.exists()) {
+                BufferedReader reader = new BufferedReader(new FileReader(file));
+                while (true) {
+                    String line = reader.readLine();
+                    if (line == null) {
+                        break;
+                    }
+                    result.append(line).append("\n");
+                }
+            } else {
+                // Does not exist:
+                return null;
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+        return result.toString();
+    }
+
     /**
      * @return Unsafe instance.
      */