8205709: Proper allocation handling

Reviewed-by: sspitsyn, mschoene, rhalade

src/java.instrument/unix/native/libinstrument/FileSystemSupport_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2018 Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
  * questions.
  */
 
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -50,6 +51,10 @@ char* basePath(const char* path) {
     } else {
         int len = last - path;
         char* str = (char*)malloc(len+1);
+        if (str == NULL) {
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
+            return NULL;
+        }
         if (len > 0) {
             memcpy(str, path, len);
         }
@@ -80,6 +85,10 @@ static char* normalizePath(const char* pathname, int len, int off) {
     if (n == 0) return strdup("/");
 
     sb = (char*)malloc(strlen(pathname)+1);
+    if (sb == NULL) {
+        fprintf(stderr, "OOM error in native tmp buffer allocation");
+        return NULL;
+    }
     sbLen = 0;
 
     if (off > 0) {
@@ -128,6 +137,10 @@ char* resolve(const char* parent, const char* child) {
     len = parentEnd + cn - childStart;
     if (child[0] == slash) {
         theChars = (char*)malloc(len+1);
+        if (theChars == NULL) {
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
+            return NULL;
+        }
         if (parentEnd > 0)
             memcpy(theChars, parent, parentEnd);
         if (cn > 0)
@@ -135,6 +148,10 @@ char* resolve(const char* parent, const char* child) {
         theChars[len] = '\0';
     } else {
         theChars = (char*)malloc(len+2);
+        if (theChars == NULL) {
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
+            return NULL;
+        }
         if (parentEnd > 0)
             memcpy(theChars, parent, parentEnd);
         theChars[parentEnd] = slash;
@@ -150,10 +167,13 @@ char* fromURIPath(const char* path) {
     if (len > 1 && path[len-1] == slash) {
         // "/foo/" --> "/foo", but "/" --> "/"
         char* str = (char*)malloc(len);
-        if (str != NULL) {
+        if (str == NULL)
-            memcpy(str, path, len-1);
+        {
-            str[len-1] = '\0';
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
+            return NULL;
         }
+        memcpy(str, path, len-1);
+        str[len-1] = '\0';
         return str;
     } else {
         return (char*)path;

src/java.instrument/windows/native/libinstrument/FileSystemSupport_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2004, 2018 Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,6 +23,7 @@
  * questions.
  */
 
+#include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
 #include <malloc.h>
@@ -66,6 +67,10 @@ char* basePath(const char* path) {
     } else {
         int len = (int)(last - path);
         char* str = (char*)malloc(len+1);
+        if (str == NULL) {
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
+            return NULL;
+        }
         if (len > 0) {
             memcpy(str, path, len);
         }
@@ -135,6 +140,10 @@ static char* normalizePath(const char* path, int len, int off) {
     if (off < 3) off = 0;       /* Avoid fencepost cases with UNC pathnames */
 
     sb = (char*)malloc(len+1);
+    if (sb == NULL) {
+        fprintf(stderr, "OOM error in native tmp buffer allocation");
+        return NULL;
+    }
     sbLen = 0;
 
     if (off == 0) {
@@ -261,11 +270,19 @@ char* resolve(const char* parent, const char* child) {
 
     if (child[childStart] == slash) {
         theChars = (char*)malloc(len+1);
+        if (theChars == NULL) {
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
+            return NULL;
+        }
         memcpy(theChars, parent, parentEnd);
         memcpy(theChars+parentEnd, child+childStart, (cn-childStart));
         theChars[len] = '\0';
     } else {
         theChars = (char*)malloc(len+2);
+        if (theChars == NULL) {
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
+            return NULL;
+        }
         memcpy(theChars, parent, parentEnd);
         theChars[parentEnd] = slash;
         memcpy(theChars+parentEnd+1, child+childStart, (cn-childStart));
@@ -320,10 +337,12 @@ char* fromURIPath(const char* path) {
         return (char*)path;
     } else {
         char* p = (char*)malloc(len+1);
-        if (p != NULL) {
+        if (p == NULL) {
-            memcpy(p, path+start, len);
+            fprintf(stderr, "OOM error in native tmp buffer allocation");
-            p[len] = '\0';
+            return NULL;
         }
+        memcpy(p, path+start, len);
+        p[len] = '\0';
         return p;
     }
 }