TIME_WAIT sockets clog system (part 3)

[Steven M. Schultz]

	Here are the latest changes in an attempt to alleviate mbuf
	exhaustion from sockets persisting in the TIME_WAIT state
	(caused by, for example, "ftp mget/mput" in a directory with
	many short files).

	There are 3 modules to be changed:  /sys/h/mbuf.h, 
	/sys/netinet/tcp_subr.c and /sys/sys/uipc_mbuf.c.  The first
	change is given as a context diff suitable for 'patch', the
	last two are replacement functions.

	The concept behind the changes is when the mbufs are exhausted
	to check whether or not the current processor priority is at
	or below the NET level (not running at interface priority) and
	use the M_DONTWAITLONG state instead of the M_DONTWAIT.  This
	insures that we will NOT sleep(), but that it is safe to call
	the drain routines (which manipulate the tcb list amoung other
	things).

	The change to mbuf.h adds the new 'wait' state and modifies the
	MGET macro.  The "mysterious" numbers 0340 and 0200 are the
	processor priority field mask and SPLNET respectively, the
	appropriate symbolic defines SHOULD have been used, but i didn't
	have the  time to futz with the necessary "ifdef/include" sequences
	to incorporate the proper header files.

	At the present time, the change to tcp_drain() only looks for
	sockets in the TIME_WAIT state to remove - this is reasonably
	safe since these are due to expire shortly anyhow.  If other
	suggestions for augmenting the tcp_drain() arrive, they can easily
	be incorporated.

	m_expand() is essentially a 4.3BSD version with an ifdef for the pdp11
	since m_clalloc() isn't implemented.

	
*** mbuf.h.old	Mon Jul  3 11:35:32 1989
--- mbuf.h	Mon Jul  3 13:50:01 1989
***************
*** 83,88 ****
--- 83,89 ----
  /* flags to m_get */
  #define	M_DONTWAIT	0
  #define	M_WAIT		1
+ #define	M_DONTWAITLONG	2
  
  /* flags to m_pgalloc */
  #define	MPG_MBUFS	0		/* put new mbufs on free list */
***************
*** 106,112 ****
  		  mfree = (m)->m_next; (m)->m_next = 0; \
  		  (m)->m_off = MMINOFF; } \
  	  else \
! 		(m) = m_more(i, t); \
  	  splx(ms); }
  /*
   * Mbuf page cluster macros.
--- 107,113 ----
  		  mfree = (m)->m_next; (m)->m_next = 0; \
  		  (m)->m_off = MMINOFF; } \
  	  else \
! 		(m) = m_more((((ms&0340) <= 0100) && (i==M_DONTWAIT)) ? M_DONTWAITLONG : i, t); \
  	  splx(ms); }
  /*
   * Mbuf page cluster macros.

==========================================================================

tcp_drain()
{
	register struct inpcb *ip, *ipnxt;
	register struct tcpcb *tp;

	/*
	 * Search through tcb's and look for TIME_WAIT states to liberate,
	 * these are due to go away soon anyhow and we're short of space or
 	 * we wouldn't be here...
	 */
	ip = tcb.inp_next;
	if (ip == 0)
		return;
	for (; ip != &tcb; ip = ipnxt) {
		ipnxt = ip->inp_next;
		tp = intotcpcb(ip);
		if (tp == 0)
			continue;
		if (tp->t_state == TCPS_TIME_WAIT)
			tcp_close(tp);
	}
}

==============================================================================

m_expand(canwait)
	int canwait;
{
	register struct domain *dp;
	register struct protosw *pr;
	register int tries;

	for (tries = 0;; ) {
#ifdef	pdp11
		if (mfree)
			return (1);
#else
		if (m_clalloc(1, MPG_MBUFS, canwait))
			return (1);
#endif
		if (canwait == M_DONTWAIT || tries++)
			return (0);

		/* ask protocols to free space */
		for (dp = domains; dp; dp = dp->dom_next)
			for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW;
			    pr++)
				if (pr->pr_drain)
					(*pr->pr_drain)();
		mbstat.m_drain++;
	}
}