From jerry@rigel.whowhere.com Wed Mar 26 10:09:41 EST 1997
Article: 31245 of news.software.nntp
Path: news.math.psu.edu!feeder.chicago.cic.net!news.radio.cz!newsbastard.radio.cz!news.radio.cz!CESspool!news.maxwell.syr.edu!news.mathworks.com!news.kei.com!news.thenet.net!uunet!in2.uu.net!205.230.7.55!whowhere.com!jerry
From: jerry@rigel.whowhere.com (Jerry Aguirre)
Newsgroups: news.software.nntp
Subject: Faster renumber patch for inn1.5.1
Date: 26 Mar 1997 03:49:05 GMT
Organization: WhoWhere?; Mt. View, CA  USA
Lines: 239
Message-ID: <5ha6bh$nph$1@fes.whowhere.com>
NNTP-Posting-Host: rigel.whowhere.com
Keywords: usenet INND renumber performance patch
Xref: news.math.psu.edu news.software.nntp:31245

Here is my fix to the lengthy time it takes innd to renumber.  I moved
finding the lowest article number to an external script and add a new
"ctlinnd setlow" command to allow setting the value for each group.  I
run news.daily with the "norenumber" option so it finishes more
quickly.  After news.daily has finished I run a "renumber" script.

Background:  The current "ctlinnd renumber" command places innd in a
state where it will read the directory for each newsgroup in the active
file to find the low and high numbered entries.  The problem is that
this can take a long time and all other innd processing is locked out.
In practical terms it can take 10 minutes to do a renumber and during
that time innd will not process articles or accept new connections for
transfer, reading, or posting.  Renumbering a group at a time helps a
little but some large groups can still lock up innd for quite a bit of
time.

When news.daily runs there is no need to find the high article number.
Innd has been maintaining that all along.  Only the low article number,
changed by the expiration of old articles, needs to be updated.  In fact
changing the high article number would be a bug!

The new "renumber" script does an ls -f of each group listed in the
active file.  It sorts that and keeps the head (lowest numberd
articles).  It then tests those one at a time until it finds a file.
(Innd will accept a sub-directory as the lowest numberd article!)
The script may take a while on a big directory but it doesn't matter as
innd can still be processing during that time.

The "renumber" script then does a "ctlinnd setlow GROUP NNN" to set the
low article number for GROUP to NNN.  Innd has little processing
involved so this completes about as fast as a "ctlinnd mode" command
would.

I have been running this for 10 days without problems.  The old renumber
command is still useful when the high article number needs to be set,
say after a crash or whenever it was out of sync.

				Jerry Aguirre

===Begin INND 1.5.1 renumber patch===
*** frontends/ctlinnd.c.orig	Tue Dec 17 06:40:40 1996
--- frontends/ctlinnd.c	Fri Mar 14 20:56:25 1997
***************
*** 99,105 ****
      {	"xabort",	"text...\t\tAbort the server",
  	1,	SC_XABORT,	TRUE	},
      {	"xexec",	"path\t\t\tExec new server",
! 	1,	SC_XEXEC,	FALSE	}
  };
  
  
--- 99,107 ----
      {	"xabort",	"text...\t\tAbort the server",
  	1,	SC_XABORT,	TRUE	},
      {	"xexec",	"path\t\t\tExec new server",
! 	1,	SC_XEXEC,	FALSE	},
!     {	"setlow",	"group number\t\tSet low article number for group",
! 	2,	SC_SETLOW,	FALSE	}
  };
  
  
*** include/inndcomm.h.orig	Tue Dec 17 06:40:40 1996
--- include/inndcomm.h	Fri Mar 14 18:34:04 1997
***************
*** 54,59 ****
--- 54,60 ----
  #define SC_TRACE	'w'
  #define SC_XABORT	'x'
  #define SC_XEXEC	'y'
+ #define SC_SETLOW	'L'
  
      /* Yes, we don't want anyone to use this. */
  #define SC_FIRSTFREE	G
*** innd/cc.c.orig	Tue Dec 17 06:40:40 1996
--- innd/cc.c	Fri Mar 14 18:36:30 1997
***************
*** 61,66 ****
--- 61,67 ----
  STATIC STRING	CCtrace();
  STATIC STRING	CCxabort();
  STATIC STRING	CCxexec();
+ STATIC STRING	CCsetlow();
  #if defined(DO_TCL)
  STATIC STRING	CCfilter();
  #endif /* defined(DO_TCL) */
***************
*** 118,124 ****
      {	SC_THROTTLE,	1, CCthrottle	},
      {	SC_TRACE,	2, CCtrace	},
      {	SC_XABORT,	1, CCxabort	},
!     {	SC_XEXEC,	1, CCxexec	}
  };
  
  STATIC SIGHANDLER	CCresetup();
--- 119,126 ----
      {	SC_THROTTLE,	1, CCthrottle	},
      {	SC_TRACE,	2, CCtrace	},
      {	SC_XABORT,	1, CCxabort	},
!     {	SC_XEXEC,	1, CCxexec	},
!     {	SC_SETLOW,	2, CCsetlow	}
  };
  
  STATIC SIGHANDLER	CCresetup();
***************
*** 1845,1848 ****
--- 1847,1878 ----
      (void)signal(s, CCresetup);
      CCclose();
      CCsetup();
+ }
+ /*
+ **  Set low number for a group in the active file.
+ */
+ STATIC STRING
+ CCsetlow(av)
+     char	*av[];
+ {
+     static char	CANTRENUMBER[] = "1 Failed (see syslog)";
+     char	*p, *p2;
+     NEWSGROUP	*ngp;
+     long	val;
+ 
+     if (Mode != OMrunning)
+ 	return CCnotrunning;
+     if (ICDneedsetup)
+ 	return "1 Must first reload newsfeeds";
+     p = av[0];
+     if (*p) p2 = av[1]; else p2 = 0;
+     if (p2) val = atol(p2);
+     else return "1 Missing or bad setlow value";
+     if (*p) {
+ 	if ((ngp = NGfind(p)) == NULL)
+ 	    return CCnogroup;
+ 	if (!NGsetlow(ngp, val))
+ 	    return CANTRENUMBER;
+     }
+     return NULL;
  }
*** innd/ng.c.orig	Tue Dec 17 06:40:40 1996
--- innd/ng.c	Fri Mar 14 18:24:44 1997
***************
*** 397,399 ****
--- 397,452 ----
      }
      return TRUE;
  }
+ 
+ /*
+ **  Set low number for a group.
+ */
+ BOOL
+ NGsetlow(ngp, val)
+     NEWSGROUP		*ngp;
+     long		val;
+ {
+     static char		NORENUMBER[] = "%s cant renumber %s %s too wide";
+     static char		RENUMBER[] = "%s renumber %s %s from %ld to %ld";
+     register char	*f2;
+     char		*f3;
+     char		*f4;
+     char		*start;
+     long		l;
+     long		himark;
+     long		lomark;
+     char		*dummy;
+ 
+     /* Get a valid offset into the active file. */
+     if (ICDneedsetup) {
+ 	syslog(L_ERROR, "%s unsynched must reload before renumber", LogName);
+ 	return FALSE;
+     }
+     start = ICDreadactive(&dummy) + ngp->Start;
+ 
+     /* Check the file format. */
+     if ((f2 = strchr(start, ' ')) == NULL
+      || (f3 = strchr(++f2, ' ')) == NULL
+      || (f4 = strchr(++f3, ' ')) == NULL) {
+ 	syslog(L_ERROR, "%s bad_format active %s",
+ 	    LogName, MaxLength(start, start));
+ 	return FALSE;
+     }
+     himark = atol(f2);
+     lomark = himark + 1;
+     if (val > lomark) {
+ 	    syslog(L_ERROR, "%s cant renumber %s %ld > %s",
+ 		    LogName, ngp->Name, val, f2);
+ 	    return FALSE;
+     }
+     l = atol(f3);
+     if (val != l) {
+ 	syslog(L_NOTICE, RENUMBER, LogName, ngp->Name, "lo", l, lomark);
+ 	if (!FormatLong(f3, val, f4 - f3)) {
+ 	    syslog(L_ERROR, NORENUMBER, LogName, ngp->Name, "lo");
+ 	    return FALSE;
+ 	}
+ 	ICDactivedirty++;
+     }
+     return TRUE;
+ }
*** /tmp/da003U3	Tue Mar 25 19:00:01 1997
--- local/renumber	Tue Mar 25 19:06:42 1997
***************
*** 0 ****
--- 1,34 ----
+ #! /bin/sh
+ ##  extracted from news.daily.  Renumber each group.  This avoids
+ #  the long outage currently caused when "ctlinnd renumber ''" is done.
+ 
+ ##  =()<. @<_PATH_SHELLVARS>@>()=
+ . /news/etc/innshellvars
+ 
+ renice 10 $$
+ if ctlinnd mode | grep 'Server running' >/dev/null ; then
+     while read GROUP hi lo flag ; do
+ 	d=`echo ${GROUP}|tr . /`
+ 	if [ -d ${SPOOL}/${d} ] ; then
+ #		Look for the lowest numbered article
+ 		a=''
+ 		for a in `ls -f ${SPOOL}/${d} | grep '^[0-9]*$' | sort -n | head -100`
+ 		do
+ 			if [ -z "$a" -o -f ${SPOOL}/${d}/$a ] ; then
+ 				break
+ 			fi
+ 		done
+ 		if [ -z "$a" ] ; then
+ #			empty newsgrup
+ 			continue
+ 		fi
+ 		until ctlinnd -s -t 90 setlow ${GROUP} $a
+ 		do
+ 		    echo -n "retrying ${GROUP} "
+ 		    sleep 10
+ 		done
+ 	fi
+     done <${ACTIVE}
+ else
+     echo 'can not renumber unless server is running'
+ fi
===End INND 1.5.1 renumber patch===