bug in malloc() under OS 3.3

[Chris Wagner]

In article <7401 at amelia.nas.nasa.gov>, plessel at prandtl.nas.nasa.gov
(Todd C. Plessel) writes:
> bug in malloc() under OS 3.3 any help would be appreciated
> =============================== cut here =================================
> /*
>  * testmem.c - test memory allocation/usage/freeing using malloc() & free()
>  *
>  * Todd Plessel
>  * NASA Ames Research Center
>  * (415) 604-4474
>  *
>  * This program demonstrates some serious bugs with the memory management
>  * rountines and/or the window manager on IRIS systems.
>  *
>  * On a 3000 or 4DG or GTX running 3.1 the following problem exists:
>  *
>  * If all the memory gets allocated, the window manager dies completely.
>  *
>  * On a 4DG or GTX running 3.2 the following problem exists:
>  *
>  * If all the memory gets allocated, the window manager kills the window
>  * (and all processes running out of it) where the program was run from.
>  *
>  * Note: both of these problems will not occur every time the program is
>  * run, but will always occur within 10 tries.
>  * Also note that ulimit(3, 0) returns a negative value (it broken).
>  *
>  *
>  * On a VGX running 3.3 the following problem exists:
>  *
>  * ulimit(3, 0) returns a bad value (about 800MB) but more importantly,
>  * malloc() allows up to about 500MB to be allocated!
>  * But when this data is actually filled, the program gets killed after
>  * about 75MB.
>  *
>  * To compile this program:
>  *
>  *        cc -o testmem testmem.c
>  *
>  *    or to use libmalloc:
>  *
>  *        cc -o testmem testmem.c -lmalloc
>  *
>  * To run:
>  *
>  *    testmem
>  *    (just press return at each prompt)
>  *
>  * (It makes no difference if we link with libmalloc or not)
>  *
>  */
> 
> /*------------------------------- INCLUDES
---------------------------------*/
> 
> #include <stdio.h>
> /* makes no difference if we use this and -lmalloc or not */
> #include <malloc.h>
> 
> /*-------------------------------- DEFINES
---------------------------------*/
> 
> /* useful byte sizes */
> 
> #define	_1B		1
> #define	_2B		2
> #define	_4B		4
> #define	_8B		8
> #define	_16B		16
> #define	_32B		32
> #define	_64B		64
> #define	_128B		128
> #define	_256B		256
> #define	_512B		512
> #define	_1KB		1024
> #define	_2KB		2048
> #define	_4KB		4096
> #define	_8KB		8192
> #define	_16KB		16384
> #define	_32KB		32768
> #define	_64KB		65536
> #define	_128KB		131072
> #define	_256KB		262144
> #define	_512KB		524288
> #define	_1MB		1048576
> #define	_2MB		2097152
> #define	_4MB		4194304
> #define	_8MB		8388608
> #define	_16MB		16777216
> #define	_32MB		33554432
> #define	_64MB		67108864
> #define	_128MB		134217728
> #define	_256MB		268435456
> #define	_512MB		536870912
> #define	_1GB		1073741824
> 
> 
> /*
>  * MAX_CHUNKS is the maximum number of mallocs that can be stored
>  * CHUNK_SIZE is the INITIAL number of bytes to request with each malloc()
>  * THRESHHOLD is the minimum number of bytes to request before giving up
>  *
>  * If CHUNK_SIZE bytes are not available then half as many bytes are
>  * requested (repeatedly) until:
>  * (1) a successful malloc occurs or
>  * (2) we have reached the THRESHHOLD - i.e., we would be requesting less
>  *     than THRESHHOLD bytes to malloc)
>  */
> 
> 
> #define	MAX_CHUNKS		5000
> #define	CHUNK_SIZE		_1MB
> #define	THRESHHOLD		_1KB
> 
> /*------------------------------- main
-------------------------------------*/
> 
> main()
> {
> 	char	*addr[MAX_CHUNKS];	/* stores allocated chunks          */
> 	int	dims[MAX_CHUNKS];	/* holds each chunk size in addr[]  */
> 	char	*p;			/* temp pointer to a[i] for filling */
> 	int	i;			/* looping index on a[]             */
> 	int	j;			/* inner loop index for filling     */
> 	int	num_chunks;		/* holds number of chunks allocated */
> 	long	chunk_size = CHUNK_SIZE;/* size (in bytes) to request       */
> 	long	threshhold = THRESHHOLD;/* min # of bytes to request        */
> 	long	total = 0;		/* total # of bytes allcoated       */
> 	char	str[80];		/* temp string for input            */
> 	int	chunk_limit = MAX_CHUNKS;/* max # of chunks to attempt      */
> 					/* this is an optional chicken exit */
> 					/* the default is NO LIMIT          */
> 
> 
> 
> 
> 
> 
> 
> 	printf("\n\n                    TESTMEM\n\n");
> 	printf(" A program for testing memory allocation/useage/freeing\n");
> 	printf(" using the standard UNIX functions malloc() and free()\n\n");
> 
> 	printf("System info:\n");
> 	printf("Memory limit        = %ld\n", ulimit(3, 0));
> 	printf("Current break point = %d\n\n", sbrk(0));
> 
> 	printf("To use the default values in [] just press <RETURN>\n\n");
> 
> 	str[0] = '\0';
> 	printf("Enter the CHUNK SIZE (# of bytes per malloc) [%ld] : ",
> 		CHUNK_SIZE);
> 	gets(str);
> 
> 	if (str[0] != '\0')
> 	{
> 		chunk_size = (long) atoi(str);
> 	
> 		if (chunk_size <= 0)
> 			chunk_size = CHUNK_SIZE;
> 	}
> 
> 	str[0] = '\0';
> 	printf("Enter the THRESHHOLD (minimum # of bytes to malloc) [%ld] : ",
> 		threshhold);
> 	gets(str);
> 
> 	if (str[0] != '\0')
> 	{
> 		threshhold = (long) atoi(str);
> 	
> 		if (threshhold <= 0)
> 			threshhold = THRESHHOLD;
> 	}
> 
> 	str[0] = '\0';
> 	printf("Enter the CHUNK LIMIT (maximum # of chunks to attempt) ");
> 	printf("[NO LIMIT] : ");
> 	gets(str);
> 
> 	if (str[0] != '\0')
> 	{
> 		chunk_limit = atoi(str);
> 	
> 		if (chunk_limit <= 0)
> 			chunk_limit = MAX_CHUNKS;
> 	}
> 
> 	printf("\n");
> 	i = 0;
> 
> 	while (1)
> 	{
> 		addr[i] = (char *) malloc (chunk_size);
> 		
> 		if (addr[i] == (char *) NULL)
> 		{
> 			if (chunk_size > threshhold)
> 			{
> 				chunk_size /= 2;
> 				continue;
> 			}
> 
> 			else
> 			{
> 				printf("Out of memory after %d mallocs.\n",
> 					i + 1);
> 				break;
> 			}
> 		}
> 
> 		printf("i = %3d (of %d): allocated %ld bytes.\n",
> 			i + 1, chunk_limit, chunk_size);
> 		total += chunk_size;
> 		dims[i] = chunk_size;
> 		++i;
> 
> 		if (i == MAX_CHUNKS)
> 		{
> 			printf("Ran out of places to store chunks!\n");
> 			printf("Recompile and/or run this program again with\n");
> 			printf("MAX_CHUNKS > %d and/or CHUNK_SIZE > %d\n",
> 				MAX_CHUNKS, chunk_size);
> 			break;
> 		}
> 
> 		if (i == chunk_limit)
> 		{
> 			printf("Quitting now while I'm ahead...\n");
> 			break;
> 		}
> 
> 	}
> 
> 	num_chunks = i;
> 
> 	printf("A total of %ld bytes were allocated in %d chunks.\n",
> 		total, num_chunks);
> 
> 	str[0] = '\0';
> 	printf("Do you want to fill the bytes with data [No] ?  ");
> 	gets(str);
> 
> 	if (str[0] == 'y' || str[0] == 'Y')
> 	{
> 		for (i = 0; i < num_chunks; ++i)
> 		{
> 			printf("i = %3d (of %d): filling %ld bytes...",
> 				i + 1, num_chunks, dims[i]);
> 
> 			p = addr[i];
> 
> 			for (j = 0; j < dims[i]; ++j)
> 				*p++ = 'A';
> 
> 			printf("\n");
> 		}
> 
> 		printf("Finished filling.\n");
> 	}
> 
> 	str[0] = '\0';
> 	printf("Do you want to free the bytes [No] ? ");
> 	gets(str);
> 
> 	if (str[0] == 'y' || str[0] == 'Y')
> 	{
> 		for (i = 0; i < num_chunks; ++i)
> 		{
> 			printf("i = %3d (of %d): freeing %ld bytes...",
> 				i + 1, num_chunks, dims[i]);
> 			free((char *) addr[i]);
> 			printf("\n");
> 		}
> 
> 		printf("Finished freeing.\n");
> 	}
> 
> 	printf("Exiting gracefully...\n\n\n");
> 
> }
> 
> 
> 
> /*------------------------- END OF FILE testmem.c
--------------------------*/

It does appear that ulimit(3) is broken on 3.3

Howver the other part isn't really broke - the system comes shipped by
default with
a MAXUMEM - the maximum virtual size of a process - at 512Mb. This
maximum can be
set lower either system wide by reconfiguring the system or per process
by using setrlimit.

We permit a process to allocate up to MAXUMEM of virtual space -
allowing a user for example
to alloc sparse matrices, or fork a large process. Only if the process
USES all the memory, then
we need a place to store it (swap space) If there is not enough swap
space, the process
is killed.  To avoid a process getting killed, reduce the maximum size
permitted.

The real bottom line here is that ulimit(3) is basically worthless - it
is a time dependent
number in any case, and of course gives one no clue as to whether the
value returned
will cause your process the thrash the main memory or not..

Chris Wagner