prims.cc (_Jv_NewObjectArray): Make sure byte size doesn't overflow a jint.

* prims.cc (_Jv_NewObjectArray): Make sure byte size doesn't
	overflow a jint.
	(_Jv_NewPrimArray): Check for overflowing a jint, replacing a
	check for overflowing size_t, since the lower level functions
	take a jint.
	* testsuite/libjava.lang/newarray_overflow.java:  New file.
	* testsuite/libjava.lang/newarray_overflow.out:  New file.

From-SVN: r72577

libjava/ChangeLog

@@ -1,3 +1,13 @@
+2003-10-17  Ralph Loader  <rcl@ihug.co.nz>
+
+	* prims.cc (_Jv_NewObjectArray): Make sure byte size doesn't
+	overflow a jint.
+	(_Jv_NewPrimArray): Check for overflowing a jint, replacing a
+	check for overflowing size_t, since the lower level functions
+	take a jint.
+	* testsuite/libjava.lang/newarray_overflow.java:  New file.
+	* testsuite/libjava.lang/newarray_overflow.out:  New file.
+	
 2003-10-15  Michael Koch  <konqueror@gmx.de>
 
 	* java/text/RuleBasedCollator.java

libjava/prims.cc

@@ -70,8 +70,10 @@ details.  */
 // around for use if we run out of memory.
 static java::lang::OutOfMemoryError *no_memory;
 
-// Largest representable size_t.
-#define SIZE_T_MAX ((size_t) (~ (size_t) 0))
+// Number of bytes in largest array object we create.  This could be
+// increased to the largest size_t value, so long as the appropriate
+// functions are changed to take a size_t argument instead of jint.
+#define MAX_OBJECT_SIZE ((1<<31) - 1)
 
 static const char *no_properties[] = { NULL };
 
@@ -481,6 +483,11 @@ _Jv_NewObjectArray (jsize count, jclass elementClass, jobject init)
   // Ensure that elements pointer is properly aligned.
   jobjectArray obj = NULL;
   size_t size = (size_t) elements (obj);
+  // Check for overflow.
+  if (__builtin_expect ((size_t) count > 
+			(MAX_OBJECT_SIZE - 1 - size) / sizeof (jobject), false))
+    throw no_memory;
+
   size += count * sizeof (jobject);
 
   jclass klass = _Jv_GetArrayClass (elementClass,
@@ -516,7 +523,7 @@ _Jv_NewPrimArray (jclass eltype, jint count)
 
   // Check for overflow.
   if (__builtin_expect ((size_t) count > 
-			(SIZE_T_MAX - size) / elsize, false))
+			(MAX_OBJECT_SIZE - size) / elsize, false))
     throw no_memory;
 
   jclass klass = _Jv_GetArrayClass (eltype, 0);

libjava/testsuite/libjava.lang/newarray_overflow.java

@@ -0,0 +1,88 @@
+/* This test checks for two slightly different overflow scenarios in
+ * array allocation.
+ *
+ * The first is that the number of bytes needed for an array size
+ * overflows on a 32 bit machine.
+ *
+ * The second is that on a 64 machine, the number of bytes silently
+ * gets truncated, resulting in too small an object being
+ * allocated.  */
+
+class newarray_overflow
+{
+  static boolean failed = false;
+
+  static void int_check()
+  {
+    int[] x;
+    try
+      {
+	x = new int [1 << 30];
+      }
+    catch (OutOfMemoryError e)
+      {
+	return;
+      }
+    /* If we really get away with it (64 bit machine), that's cool.  */
+    if (x == null) {
+      System.err.println ("int check: new returned null.");
+      failed = true;
+      return;
+    }
+    try
+      {
+	// Only check a few places so we don't thrash too badly.
+	for (int i = 0; i < x.length; i += (1 << 24))
+	  if (x[i] != 0)
+	    failed = true;
+      }
+    catch (Throwable e)
+      {
+	System.err.print ("int check: ");
+	System.err.println (e);
+	failed = true;
+      }
+  }
+
+  static void object_check()
+  {
+    Object[] x;
+    try
+      {
+	x = new Object [1 << 30];
+	System.err.println ("Alloc succeeded.");
+	System.err.println (x);
+      }
+    catch (OutOfMemoryError e)
+      {
+	return;
+      }
+    /* If we really get away with it (64 bit machine), that's cool.  */
+    if (x == null) {
+      System.err.println ("Object check: new returned null.");
+      failed = true;
+      return;
+    }
+    try
+      {
+	for (int i = 0; i < x.length; i += (1 << 24))
+	  if (x[i] != null)
+	    failed = true;
+      }
+    catch (Throwable e)
+      {
+	System.err.print ("Object check: ");
+	System.err.println (e);
+	failed = true;
+      }
+  }
+
+  public static void main (String[] ignore)
+  {
+    int_check();
+    object_check();
+
+    if (!failed)
+      System.out.println ("ok");
+  }
+}

libjava/testsuite/libjava.lang/newarray_overflow.out

@@ -0,0 +1 @@
+ok