libast: Update cdt(3): Allow empty strings in (dt)trees

This backports most of the Cdt (container data types) mechanism
from the ksh 93v- beta, based on ground work done by OpenSUSE:
https://build.opensuse.org/package/view_file/shells/ksh/ksh93-dttree-crash.dif
plus adaptations to match ksh 93u+m and an updated manual page
(src/lib/libast/man/cdt.3) added directly from the 93v- sources.

| Thu Dec 20 12:48:02 UTC 2012 - werner@suse.de
|
| - Add ksh93-dttree-crash.dif - Allow empty strings in (dt)trees
|   (bnc#795324)
|
| Fri Oct 25 14:07:57 UTC 2013 - werner@suse.de
|
| - Rework patch ksh93-dttree-crash.dif

As usual, precious little information is available because the
OpenSUSE bug report is currently closed to the public:
https://bugzilla.opensuse.org/show_bug.cgi?id=795324

However, a cursory inspection suggests that this code contains
improvements to do with concurrent processing and related
robustness. The new cdt.3 manual page adds a lot about that.

This has been in production use on OpenSUSE for a long time,
so hopefully this will make ksh a little more stable again.
Only one way to find out: let's commit and test this...

BTW, to get a nice manual, use groff and ghostscript's ps2pdf:
$ groff -tman src/lib/libast/man/cdt.3 | ps2pdf - cdt.3.pdf

src/cmd/ksh93/sh/arith.c

@@ -184,7 +184,10 @@ static Namval_t *scope(register Namval_t *np,register struct lval *lvalue,int as
 			{
 				ap = nv_arrayptr(np);
 				if(ap && !ap->table)
+				{
 					ap->table = dtopen(&_Nvdisc,Dtoset);
+					dtuserdata(ap->table,shp,1);
+				}
 				if(ap && ap->table && (nq=nv_search(nv_getsub(np),ap->table,NV_ADD)))
 					nq->nvenv = (char*)np;
 				if(nq && nv_isnull(nq))

src/cmd/ksh93/sh/array.c

@@ -95,6 +95,7 @@ static Namarr_t *array_scope(Namval_t *np, Namarr_t *ap, int flags)
 	if(is_associative(aq))
 	{
 		aq->scope = (void*)dtopen(&_Nvdisc,Dtoset);
+		dtuserdata(aq->scope,&sh,1);
 		dtview((Dt_t*)aq->scope,aq->table);
 		aq->table = (Dt_t*)aq->scope;
 		return(aq);
@@ -373,7 +374,10 @@ static Namval_t *array_find(Namval_t *np,Namarr_t *arp, int flag)
 		{
 			char *cp;
 			if(!ap->header.table)
+			{
 				ap->header.table = dtopen(&_Nvdisc,Dtoset);
+				dtuserdata(ap->header.table,&sh,1);
+			}
 			sfprintf(sh.strbuf,"%d",ap->cur);
 			cp = sfstruse(sh.strbuf);
 			mp = nv_search(cp, ap->header.table, NV_ADD);
@@ -410,7 +414,10 @@ int nv_arraysettype(Namval_t *np, Namval_t *tp, const char *sub, int flags)
 	av[1] = 0;
 	sh.last_table = 0;
 	if(!ap->table)
+	{
 		ap->table = dtopen(&_Nvdisc,Dtoset);
+		dtuserdata(ap->table,&sh,1);
+	}
 	if(nq = nv_search(sub, ap->table, NV_ADD))
 	{
 		if(!nq->nvfun && nq->nvalue.cp && *nq->nvalue.cp==0)
@@ -452,7 +459,6 @@ static Namfun_t *array_clone(Namval_t *np, Namval_t *mp, int flags, Namfun_t *fp
 	int			nelem, skipped=0;
 	Dt_t			*otable=ap->table;
 	struct index_array	*aq = (struct index_array*)ap, *ar;
-	Shell_t			*shp = sh_getinterp();
 	if(flags&NV_MOVE)
 	{
 		if((flags&NV_COMVAR) && nv_putsub(np,NIL(char*),ARRAY_SCAN))
@@ -485,6 +491,7 @@ static Namfun_t *array_clone(Namval_t *np, Namval_t *mp, int flags, Namfun_t *fp
 	if(ap->table)
 	{
 		ap->table = dtopen(&_Nvdisc,Dtoset);
+		dtuserdata(ap->table,&sh,1);
 		if(ap->scope && !(flags&NV_COMVAR))
 		{
 			ap->scope = ap->table;
@@ -855,6 +862,7 @@ static struct index_array *array_grow(Namval_t *np, register struct index_array
 		if(nv_hasdisc(np,&array_disc) || (nv_type(np) && nv_isvtree(np)))
 		{
 			ap->header.table = dtopen(&_Nvdisc,Dtoset);
+			dtuserdata(ap->header.table,&sh,1);
 			mp = nv_search("0", ap->header.table,NV_ADD);
 			if(mp && nv_isnull(mp))
 			{
@@ -1180,7 +1188,6 @@ Namval_t *nv_putsub(Namval_t *np,register char *sp,register long mode)
 	{
 		if(sp)
 		{
-			Shell_t	*shp = sh_getinterp();
 			if(ap && ap->xp && !strmatch(sp,"+([0-9])"))
 			{
 				Namval_t *mp = nv_namptr(ap->xp,0);
@@ -1188,7 +1195,7 @@ Namval_t *nv_putsub(Namval_t *np,register char *sp,register long mode)
 				size = nv_getnum(mp);
 			}
 			else
-				size = (int)sh_arith(shp,(char*)sp);
+				size = (int)sh_arith(&sh,(char*)sp);
 		}
 		if(size <0 && ap)
 			size += array_maxindex(np);
@@ -1242,7 +1249,10 @@ Namval_t *nv_putsub(Namval_t *np,register char *sp,register long mode)
 					char *cp;
 					Namval_t *mp;
 					if(!ap->header.table)
+					{
 						ap->header.table = dtopen(&_Nvdisc,Dtoset);
+						dtuserdata(ap->header.table,&sh,1);
+					}
 					sfprintf(sh.strbuf,"%d",ap->cur);
 					cp = sfstruse(sh.strbuf);
 					mp = nv_search(cp, ap->header.table, NV_ADD);
@@ -1379,7 +1389,6 @@ static void array_fixed_setdata(Namval_t *np,Namarr_t* ap,struct fixed_array* fp
 
 static int array_fixed_init(Namval_t *np, char *sub, char *cp)
 {
-	Shell_t			*shp=sh_getinterp();
 	Namarr_t		*ap;
 	struct fixed_array	*fp;
 	int			n=1,sz;
@@ -1403,12 +1412,12 @@ static int array_fixed_init(Namval_t *np, char *sub, char *cp)
 	fp->max = (int*)(fp+1);
 	fp->incr = fp->max+n;
 	fp->cur = fp->incr+n;
-	fp->max[0] = (int)sh_arith(shp,(char*)sub);
+	fp->max[0] = (int)sh_arith(&sh,(char*)sub);
 	for(n=1,ep=cp;*ep=='['; ep=cp)
 	{
 		cp = nv_endsubscript(np,ep,0);
 		cp[-1]=0;
-		fp->max[n++] = sz = (int)sh_arith(shp,(char*)ep+1);
+		fp->max[n++] = sz = (int)sh_arith(&sh,(char*)ep+1);
 		if(sz<0)
 		{
 			free((void*)ap);
@@ -1429,7 +1438,6 @@ static int array_fixed_init(Namval_t *np, char *sub, char *cp)
 
 static char *array_fixed(Namval_t *np, char *sub, char *cp,int mode)
 {
-	Shell_t			*shp=sh_getinterp();
 	Namarr_t		*ap = nv_arrayptr(np);
 	struct fixed_array	*fp = (struct fixed_array*)ap->fixed;
 	char			*ep;
@@ -1447,7 +1455,7 @@ static char *array_fixed(Namval_t *np, char *sub, char *cp,int mode)
 	}
 	else
 		fp->curi = 0;
-	size = (int)sh_arith(shp,(char*)sub);
+	size = (int)sh_arith(&sh,(char*)sub);
 	fp->cur[n] = size;
 	if(size >= fp->max[n] || (size < 0))
 		errormsg(SH_DICT,ERROR_exit(1),e_subscript, nv_name(np));
@@ -1459,7 +1467,7 @@ static char *array_fixed(Namval_t *np, char *sub, char *cp,int mode)
 			errormsg(SH_DICT,ERROR_exit(1),e_subscript, nv_name(np));
 		cp = nv_endsubscript(np,ep,0);
 		cp[-1]=0;
-		size = (int)sh_arith(shp,(char*)ep+1);
+		size = (int)sh_arith(&sh,(char*)ep+1);
 		if(size >= fp->max[n] || (size < 0))
 			errormsg(SH_DICT,ERROR_exit(1),e_subscript, nv_name(np));
 		fp->cur[n] = size;
@@ -1658,6 +1666,7 @@ void *nv_associative(register Namval_t *np,const char *sp,int mode)
 		if(ap = (struct assoc_array*)calloc(1,sizeof(struct assoc_array)))
 		{
 			ap->header.table = dtopen(&_Nvdisc,Dtoset);
+			dtuserdata(ap->header.table,&sh,1);
 			ap->cur = 0;
 			ap->pos = 0;
 			ap->header.hdr.disc = &array_disc;
@@ -1730,8 +1739,7 @@ void *nv_associative(register Namval_t *np,const char *sp,int mode)
 	    case NV_ANAME:
 		if(ap->cur)
 		{
-			Shell_t *shp = sh_getinterp();
+			if(!sh.instance && nv_isnull(ap->cur))
-			if(!shp->instance && nv_isnull(ap->cur))
 				return(NIL(void*));
 			return((void*)ap->cur->nvname);
 		}

src/cmd/ksh93/sh/init.c

@@ -1812,9 +1812,13 @@ static Init_t *nv_init(Shell_t *shp)
 	(OPTINDNOD)->nvalue.lp = (&shp->st.optindex);
 	/* set up the seconds clock */
 	shp->alias_tree = dtopen(&_Nvdisc,Dtoset);
+	dtuserdata(shp->alias_tree,shp,1);
 	shp->track_tree = dtopen(&_Nvdisc,Dtset);
+	dtuserdata(shp->track_tree,shp,1);
 	shp->bltin_tree = sh_inittree(shp,(const struct shtable2*)shtab_builtins);
+	dtuserdata(shp->bltin_tree,shp,1);
 	shp->fun_tree = dtopen(&_Nvdisc,Dtoset);
+	dtuserdata(shp->fun_tree,shp,1);
 	dtview(shp->fun_tree,shp->bltin_tree);
 	nv_mount(DOTSHNOD, "type", shp->typedict=dtopen(&_Nvdisc,Dtoset));
 	nv_adddisc(DOTSHNOD, shdiscnames, (Namval_t**)0);
@@ -1857,6 +1861,7 @@ Dt_t *sh_inittree(Shell_t *shp,const struct shtable2 *name_vals)
 		nbltins = n;
 	}
 	base_treep = treep = dtopen(&_Nvdisc,Dtoset);
+	dtuserdata(treep,shp,1);
 	treep->user = (void*)shp;
 	for(tp=name_vals;*tp->sh_name;tp++,np++)
 	{

src/cmd/ksh93/sh/macro.c

@@ -2738,7 +2738,10 @@ static char *sh_tilde(Shell_t *shp,register const char *string)
 skip:
 #endif /* _WINIX */
 	if(!logins_tree)
+	{
 		logins_tree = dtopen(&_Nvdisc,Dtbag);
+		dtuserdata(logins_tree,shp,1);
+	}
 	if(np=nv_search(string,logins_tree,NV_ADD))
 	{
 		save = shp->subshell;

src/cmd/ksh93/sh/main.c

@@ -181,6 +181,7 @@ int sh_main(int ac, char *av[], Shinit_f userinit)
 				/* preset aliases for interactive non-POSIX ksh */
 				dtclose(shp->alias_tree);
 				shp->alias_tree = sh_inittree(shp,shtab_aliases);
+				dtuserdata(shp->alias_tree,shp,1);
 			}
 		}
 #if SHOPT_REMOTE

src/cmd/ksh93/sh/name.c

@@ -789,6 +789,7 @@ Namval_t *nv_create(const char *name,  Dt_t *root, int flags, Namfun_t *dp)
 				{
 					Dt_t *dp = dtview(shp->var_tree,(Dt_t*)0);
 					rp->sdict = dtopen(&_Nvdisc,Dtoset);
+					dtuserdata(rp->sdict,shp,1);
 					dtview(rp->sdict,dp);
 					dtview(shp->var_tree,rp->sdict);
 				}
@@ -1131,7 +1132,10 @@ Namval_t *nv_create(const char *name,  Dt_t *root, int flags, Namfun_t *dp)
 								ap = nv_arrayptr(np);
 							}
 							if(n && ap && !ap->table)
+							{
 								ap->table = dtopen(&_Nvdisc,Dtoset);
+								dtuserdata(ap->table,shp,1);
+							}
 							if(ap && ap->table && (nq=nv_search(sub,ap->table,n)))
 								nq->nvenv = (char*)np;
 							if(nq && nv_isnull(nq))
@@ -2285,6 +2289,7 @@ void sh_scope(Shell_t *shp, struct argnod *envlist, int fun)
 		newroot = nv_dict(shp->namespace);
 #endif /* SHOPT_NAMESPACE */
 	newscope = dtopen(&_Nvdisc,Dtoset);
+	dtuserdata(newscope,shp,1);
 	if(envlist)
 	{
 		dtview(newscope,(Dt_t*)shp->var_tree);
@@ -3277,7 +3282,10 @@ int nv_rename(register Namval_t *np, int flags)
 		if(ap=nv_arrayptr(np))
 		{
 			if(!ap->table)
+			{
 				ap->table = dtopen(&_Nvdisc,Dtoset);
+				dtuserdata(ap->table,shp,1);
+			}
 			if(ap->table)
 				mp = nv_search(nv_getsub(np),ap->table,NV_ADD);
 			nv_arraychild(np,mp,0);

src/cmd/ksh93/sh/nvdisc.c

@@ -1299,6 +1299,7 @@ static Namfun_t *clone_table(Namval_t* np, Namval_t *mp, int flags, Namfun_t *fp
 	Dt_t		*oroot=tp->dict,*nroot=dtopen(&_Nvdisc,Dtoset);
 	if(!nroot)
 		return(0);
+	dtuserdata(nroot,dtuserdata(oroot,0,0),1);
 	memcpy((void*)ntp,(void*)fp,sizeof(struct table));
 	ntp->dict = nroot;
 	ntp->parent = nv_lastdict();

src/cmd/ksh93/sh/subshell.c

@@ -404,6 +404,7 @@ Dt_t *sh_subtracktree(int create)
 	if(create && sh.subshell && !sh.subshare && sp && !sp->strack)
 	{
 		sp->strack = dtopen(&_Nvdisc,Dtset);
+		dtuserdata(sp->strack,&sh,1);
 		dtview(sp->strack,sh.track_tree);
 		sh.track_tree = sp->strack;
 	}
@@ -420,6 +421,7 @@ Dt_t *sh_subfuntree(int create)
 	if(create && sh.subshell && !sh.subshare && sp && !sp->sfun)
 	{
 		sp->sfun = dtopen(&_Nvdisc,Dtoset);
+		dtuserdata(sp->sfun,&sh,1);
 		dtview(sp->sfun,sh.fun_tree);
 		sh.fun_tree = sp->sfun;
 	}

src/cmd/ksh93/sh/xec.c

@@ -2429,6 +2429,7 @@ int sh_exec(register const Shnode_t *t, int flags)
 				else
 				{
 					root = dtopen(&_Nvdisc,Dtoset);
+					dtuserdata(root,shp,1);
 					nv_mount(np, (char*)0, root);
 					np->nvalue.cp = Empty;
 					dtview(root,shp->var_base);
@@ -2484,11 +2485,6 @@ int sh_exec(register const Shnode_t *t, int flags)
 				slp = (struct slnod*)np->nvenv;
 				sh_funstaks(slp->slchild,-1);
 				stakdelete(slp->slptr);
-				if(shp->funload)
-				{
-					free((void*)np->nvalue.rp);
-					np->nvalue.rp = 0;
-				}
 				if(rp->sdict)
 				{
 					Namval_t *mp, *nq;
@@ -2502,6 +2498,12 @@ int sh_exec(register const Shnode_t *t, int flags)
 					dtclose(rp->sdict);
 					rp->sdict = 0;
 				}
+				if(shp->funload)
+				{
+					if(!shp->fpathdict)
+						free((void*)np->nvalue.rp);
+					np->nvalue.rp = 0;
+				}
 			}
 			if(!np->nvalue.rp)
 			{
@@ -2540,7 +2542,10 @@ int sh_exec(register const Shnode_t *t, int flags)
 					if(!shp->fpathdict)
 						shp->fpathdict = dtopen(&_Rpdisc,Dtobag);
 					if(shp->fpathdict)
+					{
+						dtuserdata(shp->fpathdict,shp,1);
 						dtinsert(shp->fpathdict,rp);
+					}
 				}
 			}
 			else

src/lib/libast/Mamfile

@@ -3964,6 +3964,14 @@ make install
 				prev cdt/dtopen.c
 				exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dtopen.c
 			done dtopen.o generated
+			make dtstat.o
+				make cdt/dtstat.c
+					prev cdt/dthdr.h implicit
+				done cdt/dtstat.c
+				meta dtstat.o %.c>%.o cdt/dtstat.c dtstat
+				prev cdt/dtstat.c
+				exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dtstat.c
+			done dtstat.o generated
 			make dtstrhash.o
 				make cdt/dtstrhash.c
 					prev cdt/dthdr.h implicit
@@ -3980,6 +3988,14 @@ make install
 				prev cdt/dttree.c
 				exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dttree.c
 			done dttree.o generated
+			make dtuser.o
+				make cdt/dtuser.c
+					prev cdt/dthdr.h implicit
+				done cdt/dtuser.c
+				meta dtuser.o %.c>%.o cdt/dtuser.c dtuser
+				prev cdt/dtuser.c
+				exec - ${CC} ${mam_cc_FLAGS} ${KSH_RELFLAGS} ${CCFLAGS} -I. -Icdt -Icomp -Iinclude -Istd -I${INSTALLROOT}/include/ast -D_PACKAGE_ast -c cdt/dtuser.c
+			done dtuser.o generated
 			make dtview.o
 				make cdt/dtview.c
 					prev cdt/dthdr.h implicit
@@ -6095,7 +6111,7 @@ make install
 			exec - ${AR} rc libast.a streval.o strexpr.o strmatch.o strcopy.o modei.o modex.o strmode.o strlcat.o strlcpy.o strlook.o strncopy.o strsearch.o strpsearch.o stresc.o stropt.o strtape.o strpcmp.o strnpcmp.o strvcmp.o strnvcmp.o tok.o tokline.o tokscan.o pathaccess.o pathcat.o pathcanon.o pathcheck.o pathpath.o pathexists.o pathfind.o pathkey.o pathprobe.o pathrepl.o pathnative.o pathposix.o pathtemp.o pathtmp.o pathstat.o pathgetlink.o pathsetlink.o pathbin.o pathshell.o pathcd.o pathprog.o ftwalk.o ftwflags.o fts.o astintercept.o conformance.o getenv.o setenviron.o optget.o optjoin.o optesc.o optctx.o strsort.o struniq.o magic.o mime.o mimetype.o signal.o sigflag.o systrace.o error.o errorf.o errormsg.o errorx.o localeconv.o setlocale.o translate.o catopen.o iconv.o lc.o lctab.o mc.o base64.o recfmt.o recstr.o reclen.o fmtrec.o fmtbase.o fmtbuf.o fmtclock.o fmtdev.o fmtelapsed.o fmterror.o fmtesc.o fmtfmt.o fmtfs.o fmtident.o fmtint.o fmtip4.o fmtip6.o fmtls.o fmtmatch.o fmtmode.o fmtnum.o fmtperm.o fmtre.o fmttime.o
 			exec - ${AR} rc libast.a fmtuid.o fmtgid.o fmtsignal.o fmtscale.o fmttmx.o fmttv.o fmtversion.o strelapsed.o strperm.o struid.o strgid.o strtoip4.o strtoip6.o stack.o stk.o swapget.o swapmem.o swapop.o swapput.o sigdata.o sigcrit.o sigunblock.o procopen.o procclose.o procrun.o procfree.o tmdate.o tmequiv.o tmfix.o tmfmt.o tmform.o tmgoff.o tminit.o tmleap.o tmlex.o tmlocale.o tmmake.o tmpoff.o tmscan.o tmsleep.o tmtime.o tmtype.o tmweek.o tmword.o tmzone.o tmxdate.o tmxduration.o tmxfmt.o tmxgettime.o tmxleap.o tmxmake.o tmxscan.o tmxsettime.o tmxsleep.o tmxtime.o tmxtouch.o tvcmp.o tvgettime.o tvsettime.o tvsleep.o tvtouch.o cmdarg.o vecargs.o vecfile.o vecfree.o vecload.o vecstring.o univdata.o touch.o mnt.o debug.o memccpy.o memchr.o memcmp.o memcpy.o memdup.o memmove.o memset.o mkdir.o mkfifo.o mknod.o rmdir.o remove.o rename.o link.o unlink.o strdup.o strchr.o strrchr.o strstr.o strtod.o strtold.o strtol.o strtoll.o strtoul.o strtoull.o strton.o strtonll.o strntod.o strntold.o strnton.o
 			exec - ${AR} rc libast.a strntonll.o strntol.o strntoll.o strntoul.o strntoull.o strcasecmp.o strncasecmp.o strerror.o mktemp.o tmpnam.o fsync.o execlp.o execve.o execvp.o execvpe.o spawnveg.o vfork.o killpg.o hsearch.o tsearch.o getlogin.o putenv.o setenv.o unsetenv.o lstat.o statvfs.o eaccess.o gross.o omitted.o readlink.o symlink.o getpgrp.o setpgid.o setsid.o waitpid.o creat64.o fcntl.o open.o atexit.o getdents.o getwd.o dup2.o errno.o getpreroot.o ispreroot.o realopen.o setpreroot.o getgroups.o mount.o system.o iblocks.o modedata.o tmdata.o memfatal.o sfkeyprintf.o sfdcdio.o sfdcdos.o sfdcfilter.o sfdcseekable.o sfdcslow.o sfdcsubstr.o sfdctee.o sfdcunion.o sfdcmore.o sfdcprefix.o wc.o wc2utf8.o basename.o closelog.o dirname.o fmtmsglib.o fnmatch.o ftw.o getdate.o getsubopt.o glob.o nftw.o openlog.o re_comp.o resolvepath.o realpath.o regcmp.o regexp.o setlogmask.o strftime.o strptime.o swab.o syslog.o tempnam.o wordexp.o mktime.o regalloc.o regclass.o regcoll.o regcomp.o regcache.o regdecomp.o regerror.o regexec.o regfatal.o reginit.o
-			exec - ${AR} rc libast.a regnexec.o regsubcomp.o regsubexec.o regsub.o regrecord.o regrexec.o regstat.o dtclose.o dtdisc.o dthash.o dtlist.o dtmethod.o dtopen.o dtstrhash.o dttree.o dtview.o dtwalk.o dtnew.o dtcomp.o sfclose.o sfclrlock.o sfdisc.o sfdlen.o sfexcept.o sfgetl.o sfgetu.o sfcvt.o sfecvt.o sffcvt.o sfextern.o sffilbuf.o sfflsbuf.o sfprints.o sfgetd.o sfgetr.o sfllen.o sfmode.o sfmove.o sfnew.o sfpkrd.o sfnotify.o sfnputc.o sfopen.o sfpeek.o sfpoll.o sfpool.o sfpopen.o sfprintf.o sfputd.o sfputl.o sfputr.o sfputu.o sfrd.o sfread.o sfreserve.o sfscanf.o sfseek.o sfset.o sfsetbuf.o sfsetfd.o sfsize.o sfsk.o sfstack.o sfstrtod.o sfsync.o sfswap.o sftable.o sftell.o sftmp.o sfungetc.o sfvprintf.o sfvscanf.o sfwr.o sfwrite.o sfpurge.o sfraise.o sfwalk.o sfgetm.o sfmutex.o sfputm.o sfresize.o _sfclrerr.o _sfeof.o _sferror.o _sffileno.o _sfopen.o _sfstacked.o _sfvalue.o _sfgetc.o _sfgetl.o _sfgetl2.o _sfgetu.o _sfgetu2.o _sfdlen.o _sfllen.o _sfslen.o _sfulen.o _sfputc.o _sfputd.o _sfputl.o _sfputm.o
+			exec - ${AR} rc libast.a regnexec.o regsubcomp.o regsubexec.o regsub.o regrecord.o regrexec.o regstat.o dtclose.o dtdisc.o dthash.o dtlist.o dtmethod.o dtopen.o dtstat.o dtstrhash.o dttree.o dtuser.o dtview.o dtwalk.o dtnew.o dtcomp.o sfclose.o sfclrlock.o sfdisc.o sfdlen.o sfexcept.o sfgetl.o sfgetu.o sfcvt.o sfecvt.o sffcvt.o sfextern.o sffilbuf.o sfflsbuf.o sfprints.o sfgetd.o sfgetr.o sfllen.o sfmode.o sfmove.o sfnew.o sfpkrd.o sfnotify.o sfnputc.o sfopen.o sfpeek.o sfpoll.o sfpool.o sfpopen.o sfprintf.o sfputd.o sfputl.o sfputr.o sfputu.o sfrd.o sfread.o sfreserve.o sfscanf.o sfseek.o sfset.o sfsetbuf.o sfsetfd.o sfsize.o sfsk.o sfstack.o sfstrtod.o sfsync.o sfswap.o sftable.o sftell.o sftmp.o sfungetc.o sfvprintf.o sfvscanf.o sfwr.o sfwrite.o sfpurge.o sfraise.o sfwalk.o sfgetm.o sfmutex.o sfputm.o sfresize.o _sfclrerr.o _sfeof.o _sferror.o _sffileno.o _sfopen.o _sfstacked.o _sfvalue.o _sfgetc.o _sfgetl.o _sfgetl2.o _sfgetu.o _sfgetu2.o _sfdlen.o _sfllen.o _sfslen.o _sfulen.o _sfputc.o _sfputd.o _sfputl.o _sfputm.o
 			exec - ${AR} rc libast.a _sfputu.o clearerr.o fclose.o fdopen.o feof.o ferror.o fflush.o fgetc.o fgetpos.o fgets.o fileno.o fopen.o fprintf.o fpurge.o fputc.o fputs.o fread.o freopen.o fscanf.o fseek.o fseeko.o fsetpos.o ftell.o ftello.o fwrite.o flockfile.o ftrylockfile.o funlockfile.o getc.o getchar.o getw.o pclose.o popen.o printf.o putc.o putchar.o puts.o putw.o rewind.o scanf.o setbuf.o setbuffer.o setlinebuf.o setvbuf.o snprintf.o sprintf.o sscanf.o asprintf.o vasprintf.o tmpfile.o ungetc.o vfprintf.o vfscanf.o vprintf.o vscanf.o vsnprintf.o vsprintf.o vsscanf.o _doprnt.o _doscan.o _filbuf.o _flsbuf.o _stdfun.o _stdopen.o _stdprintf.o _stdscanf.o _stdsprnt.o _stdvbuf.o _stdvsnprnt.o _stdvsprnt.o _stdvsscn.o fgetwc.o fwprintf.o putwchar.o vfwscanf.o wprintf.o fgetws.o fwscanf.o swprintf.o vswprintf.o wscanf.o fputwc.o getwc.o swscanf.o vswscanf.o fputws.o getwchar.o ungetwc.o vwprintf.o fwide.o putwc.o vfwprintf.o vwscanf.o stdio_c99.o fcloseall.o fmemopen.o getdelim.o getline.o frexp.o frexpl.o astcopy.o
 			exec - ${AR} rc libast.a astconf.o astdynamic.o astlicense.o astquery.o astwinsize.o conftab.o aststatic.o getopt.o getoptl.o aso.o asolock.o asometh.o asorelax.o aso-sem.o aso-fcntl.o vmbest.o vmclear.o vmclose.o vmdcheap.o vmdebug.o vmdisc.o vmexit.o vmlast.o vmopen.o vmpool.o vmprivate.o vmprofile.o vmregion.o vmsegment.o vmset.o vmstat.o vmstrdup.o vmtrace.o vmwalk.o vmmopen.o malloc.o vmgetmem.o a64l.o acosh.o asinh.o atanh.o cbrt.o crypt.o erf.o err.o exp.o exp__E.o expm1.o gamma.o getpass.o lgamma.o log.o log1p.o log__L.o rand48.o random.o rcmd.o rint.o support.o sfstrtmp.o spawn.o
 			exec - (ranlib libast.a) >/dev/null 2>&1 || true

src/lib/libast/cdt/cdtlib.h

@@ -58,9 +58,9 @@
 /* This struct holds private method data created on DT_OPEN */
 struct _dtdata_s
 {	unsigned int	lock;	/* general dictionary lock	*/
-	Dtuser_t	user;	/* application's data		*/
 	unsigned int	type;	/* method type, control flags	*/
 	ssize_t		size;	/* number of objects		*/
+	Dtuser_t	user;	/* application's data		*/
 	Dt_t		dict;	/* when DT_INDATA is requested	*/
 };
 
@@ -123,7 +123,7 @@ typedef struct _dtlib_s
 #endif /* _BLD_cdt */
 
 /* these macros lock/unlock dictionaries. DTRETURN substitutes for "return" */
-#define DTSETLOCK(dt)		(((dt)->data->type&DT_SHARE) ? asolock(&(dt)->data->lock,1,ASO_SPINLOCK) : 0 )
+#define DTSETLOCK(dt)		(((dt)->data->type&DT_SHARE) ? asolock(&(dt)->data->lock,1,ASO_LOCK) : 0 )
 #define DTCLRLOCK(dt)		(((dt)->data->type&DT_SHARE) ? asolock(&(dt)->data->lock,1,ASO_UNLOCK) : 0 )
 #define DTRETURN(ob,rv)		do { (ob) = (rv); goto dt_return; } while(0)
 #define DTERROR(dt, mesg) 	(!((dt)->disc && (dt)->disc->eventf) ? 0 : \

src/lib/libast/cdt/dtcomp.c

@@ -52,9 +52,3 @@ extern ssize_t dtsize(Dt_t* d)
 {
 	return (ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_STAT);
 }
-
-#undef dtstat
-extern ssize_t dtstat(Dt_t* d)
-{
-	return (ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_STAT);
-}

src/lib/libast/cdt/dthash.c

@@ -52,11 +52,13 @@ static int htable(Dt_t* dt)
 	if((n = hash->tblz) > 0 && (hash->type&H_FIXED) )
 		return 0; /* fixed size table */
 
-	if(n == 0 && disc && disc->eventf) /* let user have input */
+	if(disc && disc->eventf) /* let user have input */
 	{	if((*disc->eventf)(dt, DT_HASHSIZE, &n, disc) > 0 )
 		{	if(n < 0) /* fix table size */
 			{	hash->type |= H_FIXED;
-				n = -n;
+				n = -n; /* desired table size */
+				if(hash->tblz >= n ) /* table size is fixed now */
+					return 0;
 			}
 		}
 	}
@@ -234,12 +236,13 @@ static Void_t* hstat(Dt_t* dt, Dtstat_t* st)
 
 		for(endt = (t = hash->htbl) + hash->tblz; t < endt; ++t)
 		{	for(n = 0, l = *t; l; l = l->_rght)
+			{	if(n < DT_MAXSIZE)
+					st->lsize[n] += 1;
 				n += 1;
+			}
 			st->mlev = n > st->mlev ? n : st->mlev;
 			if(n < DT_MAXSIZE) /* if chain length is small */
-			{	st->msize = n > st->msize ? n : st->msize;
+				st->msize = n > st->msize ? n : st->msize;
-				st->lsize[n] += n;
-			}
 		}
 	}
 
@@ -310,7 +313,7 @@ int	type;
 	hsh = _DTHSH(dt,key,disc);
 
 	tbl = hash->htbl + (hsh & (hash->tblz-1));
-	pp = ll = NIL(Dtlink_t*);
+	pp = ll = NIL(Dtlink_t*); /* pp is the before, ll is the here */
 	for(p = NIL(Dtlink_t*), l = *tbl; l; p = l, l = l->_rght)
 	{	if(hsh == l->_hash)
 		{	o = _DTOBJ(disc,l); k = _DTKEY(disc,o);
@@ -342,20 +345,41 @@ int	type;
 			_dtfree(dt, ll, type);
 			DTRETURN(obj, _DTOBJ(disc,ll));
 		}
+		else if(type & DT_INSTALL )
+		{	if(dt->meth->type&DT_BAG)
+				goto do_insert;
+			else if(!(lnk = _dtmake(dt, obj, type)) )
+				DTRETURN(obj, NIL(Void_t*) );
+			else /* replace old object with new one */
+			{	if(pp) /* remove old object */
+					pp->_rght = ll->_rght;
+				else	*tbl = ll->_rght;
+				o = _DTOBJ(disc,ll);
+				_dtfree(dt, ll, DT_DELETE);
+				DTANNOUNCE(dt, o, DT_DELETE);
+
+				goto do_insert;
+			}
+		}
 		else
 		{	/**/DEBUG_ASSERT(type&(DT_INSERT|DT_ATTACH|DT_APPEND|DT_RELINK));
-			if(!(dt->meth->type&DT_BAG) )
+			if((dt->meth->type&DT_BAG) )
+				goto do_insert;
+			else
 			{	if(type&(DT_INSERT|DT_APPEND|DT_ATTACH) )
-					type |= DT_SEARCH; /* for announcement */
+					type |= DT_MATCH; /* for announcement */
 				else if(lnk && (type&DT_RELINK) )
+				{	/* remove a duplicate */
+					o = _DTOBJ(disc, lnk);
 					_dtfree(dt, lnk, DT_DELETE);
+					DTANNOUNCE(dt, o, DT_DELETE);
+				}
 				DTRETURN(obj, _DTOBJ(disc,ll));
 			}
-			else	goto do_insert;
 		}
 	}
 	else /* no matching object */
-	{	if(!(type&(DT_INSERT|DT_APPEND|DT_ATTACH|DT_RELINK)) )
+	{	if(!(type&(DT_INSERT|DT_INSTALL|DT_APPEND|DT_ATTACH|DT_RELINK)) )
 			DTRETURN(obj, NIL(Void_t*));
 
 	do_insert: /* inserting a new object */

src/lib/libast/cdt/dtlist.c

@@ -142,9 +142,9 @@ int		type;
 }
 
 #if __STD_C
-static Void_t* liststat(Dt_t* dt, Dtstat_t* st)
+static Void_t* listat(Dt_t* dt, Dtstat_t* st)
 #else
-static Void_t* liststat(dt, st)
+static Void_t* listat(dt, st)
 Dt_t*		dt;
 Dtstat_t*	st;
 #endif
@@ -186,7 +186,7 @@ int	type;
 	else if(type&DT_CLEAR)
 		DTRETURN(obj, lclear(dt));
 	else if(type&DT_STAT )
-		DTRETURN(obj, liststat(dt, (Dtstat_t*)obj));
+		DTRETURN(obj, listat(dt, (Dtstat_t*)obj));
 
 	h = list->here; /* save finger to last search object */
 	list->here = NIL(Dtlink_t*);
@@ -202,8 +202,9 @@ int	type;
 	{	r = (Dtlink_t*)obj;
 		goto do_insert;
 	}
-	else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH))
+	else if(type&(DT_INSERT|DT_INSTALL|DT_APPEND|DT_ATTACH))
-	{	if(!(r = _dtmake(dt, obj, type)) )
+	{ dt_insert:
+		if(!(r = _dtmake(dt, obj, type)) )
 			DTRETURN(obj, NIL(Void_t*));
 		dt->data->size += 1;
 
@@ -290,7 +291,7 @@ int	type;
 		}
 		r = h ? h : r;
 	}
-	if(!r)
+	if(!r) /* not found */
 		DTRETURN(obj, NIL(Void_t*));
 
 	if(type&(DT_DELETE|DT_DETACH|DT_REMOVE))

src/lib/libast/cdt/dtopen.c

@@ -153,25 +153,3 @@ void _dtfree(Dt_t* dt, Dtlink_t* l, int type)
 	if(disc->link < 0) /* free holder */
 		(void)(*dt->memoryf)(dt, (Void_t*)l, 0, disc);
 }
-
-int dtuserlock(Dt_t* dt, unsigned int key, int type)
-{
-	if(type > 0)
-		return asolock(&dt->data->user.lock, key, ASO_LOCK);
-	else if(type < 0)
-		return asolock(&dt->data->user.lock, key, ASO_UNLOCK);
-	else	return asolock(&dt->data->user.lock, key, ASO_TRYLOCK);
-}
-
-Void_t* dtuserdata(Dt_t* dt, Void_t* data, unsigned int key)
-{
-	if(key == 0)
-		return dt->data->user.data;
-	else if(dtuserlock(dt, key, 1) < 0 )
-		return NIL(Void_t*);
-	else
-	{	dt->data->user.data = data;
-		dtuserlock(dt, key, -1);
-		return data;
-	}
-}

src/lib/libast/cdt/dtstat.c

@@ -0,0 +1,54 @@
+/***********************************************************************
+*                                                                      *
+*               This software is part of the ast package               *
+*          Copyright (c) 1985-2012 AT&T Intellectual Property          *
+*                      and is licensed under the                       *
+*                 Eclipse Public License, Version 1.0                  *
+*                    by AT&T Intellectual Property                     *
+*                                                                      *
+*                A copy of the License is available at                 *
+*          http://www.eclipse.org/org/documents/epl-v10.html           *
+*         (with md5 checksum b35adb5213ca9657e911e9befb180842)         *
+*                                                                      *
+*              Information and Software Systems Research               *
+*                            AT&T Research                             *
+*                           Florham Park NJ                            *
+*                                                                      *
+*                 Glenn Fowler <gsf@research.att.com>                  *
+*                  David Korn <dgk@research.att.com>                   *
+*                   Phong Vo <kpv@research.att.com>                    *
+*                                                                      *
+***********************************************************************/
+#include	"dthdr.h"
+
+/* Get statistics for a dictionary
+**
+** Written by Kiem-Phong Vo
+*/
+
+ssize_t dtstat(Dt_t* dt, Dtstat_t* dtst)
+{
+	ssize_t	sz, k, maxk;
+	char	*str;
+	char	*end;
+
+	sz = (ssize_t)(*dt->meth->searchf)(dt, (Void_t*)dtst, DT_STAT);
+
+	str = dtst->mesg;
+	end = &dtst->mesg[sizeof(dtst->mesg)] - 1;
+	str += sfsprintf(str, end - str, "Objects=%d Levels=%d(Largest:", dtst->size, dtst->mlev+1);
+
+	/* print top 3 levels */
+	for(k = maxk = 0; k <= dtst->mlev; ++k)
+		if(dtst->lsize[k] > dtst->lsize[maxk])
+			maxk = k;
+	if(maxk > 0)
+		maxk -= 1;
+	for(k = 0; k < 3 && maxk <= dtst->mlev; ++k, ++maxk)
+		str += sfsprintf(str, end - str, " lev[%d]=%d", maxk, dtst->lsize[maxk] );
+	if (str < end)
+		*str++ = ')';
+	*str = 0;
+
+	return sz;
+}

src/lib/libast/cdt/dtstrhash.c

@@ -22,40 +22,38 @@
 #include	"dthdr.h"
 
 /* Hashing a string into an unsigned integer.
-** The basic method is to continuingly accumulate bytes and multiply
+** This is the FNV (Fowler-Noll-Vo) hash function.
-** with some given prime. The length n of the string is added last.
+** Written by Kiem-Phong Vo (01/10/2012)
-** The recurrent equation is like this:
-**	h[k] = (h[k-1] + bytes)*prime	for 0 <= k < n
-**	h[n] = (h[n-1] + n)*prime
-** The prime is chosen to have a good distribution of 1-bits so that
-** the multiplication will distribute the bits in the accumulator well.
-** The below code accumulates 2 bytes at a time for speed.
-**
-** Written by Kiem-Phong Vo (02/28/03)
 */
 
 #if __STD_C
 uint dtstrhash(uint h, Void_t* args, ssize_t n)
 #else
 uint dtstrhash(h,args,n)
-reg uint	h;
+uint	h;
 Void_t*		args;
 ssize_t		n;
 #endif
 {
 	unsigned char	*s = (unsigned char*)args;
 
-	if(n <= 0)
+#if _ast_sizeof_int == 8 /* 64-bit hash */
-	{	for(; *s != 0; s += s[1] ? 2 : 1)
+#define	FNV_PRIME	((1<<40) + (1<<8) + 0xb3)
-			h = (h + (s[0]<<8) + s[1])*DT_PRIME;
+#define FNV_OFFSET	14695981039346656037
-		n = s - (unsigned char*)args;
+#else /* 32-bit hash */
+#define	FNV_PRIME	((1<<24) + (1<<8) + 0x93)
+#define FNV_OFFSET	2166136261
+#endif
+	h = (h == 0 || h == ~0) ? FNV_OFFSET : h;
+	if(n <= 0) /* see discipline key definition for == 0 */
+	{	for(; *s != 0; ++s )
+			h = (h ^ s[0]) * FNV_PRIME;
 	}
 	else
 	{	unsigned char*	ends;
-		for(ends = s+n-1; s < ends; s += 2)
+		for(ends = s+n; s < ends; ++s)
-			h = (h + (s[0]<<8) + s[1])*DT_PRIME;
+			h = (h ^ s[0]) * FNV_PRIME;
-		if(s <= ends)
-			h = (h + (s[0]<<8))*DT_PRIME;
 	}
-	return (h+n)*DT_PRIME;
+
+	return h;
 }

src/lib/libast/cdt/dttree.c

@@ -545,7 +545,14 @@ int		type;
 			}
 			else	goto no_root;
 		}
-		else if(type&DT_REMOVE) /* remove a particular element in the tree */
+		else if(type&(DT_DELETE|DT_DETACH))
+		{ dt_delete: /* remove an object from the dictionary */
+			obj = _DTOBJ(disc,root);
+			_dtfree(dt, root, type);
+			dt->data->size -= 1;
+			goto no_root;
+		}
+		else if(type&DT_REMOVE) /* remove a particular object */
 		{	if(_DTOBJ(disc,root) == obj)
 				goto dt_delete;
 			else
@@ -555,28 +562,32 @@ int		type;
 				DTRETURN(obj, NIL(Void_t*));
 			}
 		}
-		else if(type&(DT_DELETE|DT_DETACH))
-		{ dt_delete: /* remove an object from the dictionary */
-			obj = _DTOBJ(disc,root);
-			_dtfree(dt, root, type);
-			dt->data->size -= 1;
-			goto no_root;
-		}
 		else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH))
 		{	if(dt->meth->type&DT_OSET)
-			{	type |= DT_SEARCH; /* for announcement */
+			{	type |= DT_MATCH; /* for announcement */
 				goto has_root;
 			}
-			else
+			else /* if(dt->meth->type&DT_OBAG) */
 			{	root->_left = NIL(Dtlink_t*);
 				root->_rght = link._left;
 				link._left = root;
 				goto dt_insert;
 			}
 		}
+		else if(type&DT_INSTALL)
+		{	/* remove old object before insert new one */
+			o = _DTOBJ(disc, root);
+			_dtfree(dt, root, DT_DELETE);
+			DTANNOUNCE(dt, o, DT_DELETE);
+			goto dt_insert;
+		}
 		else if(type&DT_RELINK) /* a duplicate */
 		{	if(dt->meth->type&DT_OSET)
+			{	/* remove object */
+				o = _DTOBJ(disc, me);
 				_dtfree(dt, me, DT_DELETE);
+				DTANNOUNCE(dt, o, DT_DELETE);
+			}
 			else
 			{	me->_left = NIL(Dtlink_t*);
 				me->_rght = link._left;
@@ -612,7 +623,7 @@ int		type;
 		{	obj = NIL(Void_t*);
 			goto no_root;
 		}
-		else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH))
+		else if(type&(DT_INSERT|DT_APPEND|DT_ATTACH|DT_INSTALL))
 		{ dt_insert:
 			if(!(root = _dtmake(dt, obj, type)) )
 			{	obj = NIL(Void_t*);

src/lib/libast/cdt/dtuser.c

@@ -0,0 +1,59 @@
+/***********************************************************************
+*                                                                      *
+*               This software is part of the ast package               *
+*          Copyright (c) 1985-2012 AT&T Intellectual Property          *
+*                      and is licensed under the                       *
+*                 Eclipse Public License, Version 1.0                  *
+*                    by AT&T Intellectual Property                     *
+*                                                                      *
+*                A copy of the License is available at                 *
+*          http://www.eclipse.org/org/documents/epl-v10.html           *
+*         (with md5 checksum b35adb5213ca9657e911e9befb180842)         *
+*                                                                      *
+*              Information and Software Systems Research               *
+*                            AT&T Research                             *
+*                           Florham Park NJ                            *
+*                                                                      *
+*                 Glenn Fowler <gsf@research.att.com>                  *
+*                  David Korn <dgk@research.att.com>                   *
+*                   Phong Vo <kpv@research.att.com>                    *
+*                                                                      *
+***********************************************************************/
+#include	"dthdr.h"
+
+/* Perform various functions on the user's behalf.
+**
+** Written by Kiem-Phong Vo (01/05/2012)
+*/
+
+/* managing the lock dt->data->user.lock */
+int dtuserlock(Dt_t* dt, unsigned int key, int type)
+{
+	if(key == 0)
+		return -1;
+	else if(type > 0)
+		return asolock(&dt->data->user.lock, key, ASO_LOCK);
+	else if(type < 0)
+		return asolock(&dt->data->user.lock, key, ASO_UNLOCK);
+	else	return asolock(&dt->data->user.lock, key, ASO_TRYLOCK);
+}
+
+/* managing the user data slot dt->data->user.data */
+Void_t* dtuserdata(Dt_t* dt, Void_t* data, int set)
+{
+	if(set == 0) /* just return current value */
+		return asogetptr(&dt->data->user.data);
+	else while(1)
+	{	Void_t	*current = dt->data->user.data;
+		if(asocasptr(&dt->data->user.data, current, data) == current)
+			return	current;
+	}
+}
+
+/* announcing an event on the user's behalf */
+int dtuserevent(Dt_t* dt, int flags, Void_t* data)
+{
+	if(!dt->disc->eventf)
+		return 0;
+	else	return (*dt->disc->eventf)(dt, DT_ANNOUNCE|DT_USER|flags, data, dt->disc);
+}

src/lib/libast/include/cdt.h

@@ -164,6 +164,7 @@ struct _dtstat_s
 	ssize_t		msize;	/* max #defined elts in below arrays	*/
 	ssize_t		lsize[DT_MAXSIZE]; /* #objects by level		*/
 	ssize_t		tsize[DT_MAXSIZE]; /* #tables by level		*/
+	char		mesg[256]; /* digest of top level statistics	*/
 };
 
 /* supported storage methods */
@@ -199,7 +200,8 @@ struct _dtstat_s
 #define DT_ATLEAST	0000040000 /* find the least elt >= object	*/
 #define DT_ATMOST	0000100000 /* find the biggest elt <= object	*/
 #define DT_REMOVE	0002000000 /* remove a specific object		*/
-#define DT_TOANNOUNCE	(DT_INSERT|DT_DELETE|DT_SEARCH|DT_NEXT|DT_PREV|DT_FIRST|DT_LAST|DT_MATCH|DT_ATTACH|DT_DETACH|DT_APPEND|DT_ATLEAST|DT_ATMOST|DT_REMOVE)
+#define DT_INSTALL      0004000000 /* install a new object		*/
+#define DT_TOANNOUNCE	(DT_INSERT|DT_DELETE|DT_SEARCH|DT_NEXT|DT_PREV|DT_FIRST|DT_LAST|DT_MATCH|DT_ATTACH|DT_DETACH|DT_APPEND|DT_ATLEAST|DT_ATMOST|DT_REMOVE|DT_INSTALL)
 
 #define DT_RELINK	0000002000 /* re-inserting (dtdisc,dtmethod...)	*/
 #define DT_FLATTEN	0000000040 /* flatten objects into a list	*/
@@ -216,6 +218,7 @@ struct _dtstat_s
 				   /* the actual event will be this bit */
 				   /* combined with the operation bit	*/
 #define DT_OPTIMIZE	0100000000 /* optimizing data structure		*/
+#define DT_USER		0200000000 /* an announcement on user's behalf	*/
 
 /* events for discipline and method event-handling functions */
 #define DT_OPEN		1	/* a dictionary is being opened		*/
@@ -277,7 +280,8 @@ extern int		dtwalk _ARG_((Dt_t*, int(*)(Dt_t*,Void_t*,Void_t*), Void_t*));
 extern int		dtcustomize _ARG_((Dt_t*, int, int));
 extern unsigned int	dtstrhash _ARG_((unsigned int, Void_t*, ssize_t));
 extern int		dtuserlock _ARG_((Dt_t*, unsigned int, int));
-extern Void_t*		dtuserdata _ARG_((Dt_t*, Void_t*, unsigned int));
+extern Void_t*		dtuserdata _ARG_((Dt_t*, Void_t*, int));
+extern int		dtuserevent _ARG_((Dt_t*, int, Void_t*));
 
 /* deal with upward binary compatibility (operation bit translation, etc.) */
 extern Dt_t*		_dtopen _ARG_((Dtdisc_t*, Dtmethod_t*, unsigned long));
@@ -334,6 +338,7 @@ _END_EXTERNS_
 #define dtsearch(d,o)	(*(_DT(d)->searchf))((d),(Void_t*)(o),DT_SEARCH)
 #define dtmatch(d,o)	(*(_DT(d)->searchf))((d),(Void_t*)(o),DT_MATCH)
 #define dtinsert(d,o)	(*(_DT(d)->searchf))((d),(Void_t*)(o),DT_INSERT)
+#define dtinstall(d,o)	(*(_DT(d)->searchf))((d),(Void_t*)(o),DT_INSTALL)
 #define dtappend(d,o)	(*(_DT(d)->searchf))((d),(Void_t*)(o),DT_APPEND)
 #define dtdelete(d,o)	(*(_DT(d)->searchf))((d),(Void_t*)(o),DT_DELETE)
 #define dtremove(d,o)	(*(_DT(d)->searchf))((d),(Void_t*)(o),DT_REMOVE)
@@ -345,7 +350,6 @@ _END_EXTERNS_
 #define dtextract(d)	(Dtlink_t*)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_EXTRACT)
 #define dtrestore(d,l)	(Dtlink_t*)(*(_DT(d)->searchf))((d),(Void_t*)(l),DT_RESTORE)
 
-#define dtstat(d,s)	(ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(s),DT_STAT)
 #define dtsize(d)	(ssize_t)(*(_DT(d)->searchf))((d),(Void_t*)(0),DT_STAT)
 
 #define DT_PRIME	17109811 /* 2#00000001 00000101 00010011 00110011 */

src/lib/libast/man/cdt.3

@@ -33,6 +33,7 @@ Dtdisc_t;
 Dtmethod_t;
 Dtlink_t;
 Dtstat_t;
+Dtuser_t;
 .Ce
 .Ss "DICTIONARY CONTROL"
 .Cs
@@ -42,10 +43,10 @@ void        dtclear(dt);
 Dtdisc_t*   dtdisc(Dt_t* dt, const Dtdisc_t* disc, int type);
 Dtmethod_t* dtmethod(Dt_t* dt, const Dtmethod_t* meth);
 Dt_t*       dtview(Dt_t* dt, Dt_t* view);
-int         dtcustomize(Dt_t* dt, int type, Void_t* arg);
+int         dtcustomize(Dt_t* dt, int type, int action);
-int         dtoptimize(Dt_t* dt);
+int         dtuserlock(Dt_t* dt, unsigned int key, int action);
-int         dtshare(Dt_t* dt, int type);
+Void_t*     dtuserdata(Dt_t* dt, Void_t* data, int set);
-int         dtlock(Dt_t* dt, unsigned int key, int type);
+int         dtuserevent(Dt_t* dt, int flags, Void_t* data);
 .Ce
 .Ss "STORAGE METHODS"
 .Cs
@@ -74,8 +75,10 @@ typedef int          (*Dtevent_f)(Dt_t*, int, Void_t*, Dtdisc_t*);
 .Ss "OBJECT OPERATIONS"
 .Cs
 Void_t*   dtinsert(Dt_t* dt, Void_t* obj);
+Void_t*   dtinstall(Dt_t* dt, Void_t* obj);
 Void_t*   dtappend(Dt_t* dt, Void_t* obj);
 Void_t*   dtdelete(Dt_t* dt, Void_t* obj);
+Void_t*   dtremove(Dt_t* dt, Void_t* obj);
 Void_t*   dtattach(Dt_t* dt, Void_t* obj);
 Void_t*   dtdetach(Dt_t* dt, Void_t* obj);
 Void_t*   dtsearch(Dt_t* dt, Void_t* obj);
@@ -84,9 +87,12 @@ Void_t*   dtfirst(Dt_t* dt);
 Void_t*   dtnext(Dt_t* dt, Void_t* obj);
 Void_t*   dtlast(Dt_t* dt);
 Void_t*   dtprev(Dt_t* dt, Void_t* obj);
-Void_t*   dtleast(Dt_t* dt, Void_t* obj);
+Void_t*   dtatleast(Dt_t* dt, Void_t* obj);
-Void_t*   dtmost(Dt_t* dt, Void_t* obj);
+Void_t*   dtatmost(Dt_t* dt, Void_t* obj);
 int       dtwalk(Dt_t* dt, int (*userf)(Dt_t*, Void_t*, Void_t*), Void_t*);
+Void_t*   dtstart(Dt_t* dt, Void_t* obj);
+Void_t*   dtstep(Dt_t* dt, Void_t* path);
+Void_t*   dtstop(Dt_t* dt, Void_t* path);
 Dtlink_t* dtflatten(Dt_t* dt);
 Dtlink_t* dtlink(Dt_t* dt, Dtlink_t* link);
 Void_t*   dtobj(Dt_t* dt, Dtlink_t* link);
@@ -101,10 +107,9 @@ Dt_t*     dtvhere(Dt_t* dt);
 ssize_t   dtsize(Dt_t* dt);
 ssize_t   dtstat(Dt_t* dt, Dtstat_t* st);
 .Ce
-.Ss "HASH FUNCTIONS"
+.Ss "HASH FUNCTION"
 .Cs
 unsigned int dtstrhash(unsigned int h, char* str, int n);
-unsigned int dtcharhash(unsigned int h, unsigned char c);
 .Ce
 .SH DESCRIPTION
 .PP
@@ -120,6 +125,7 @@ and \f5char\fP for older C compilation environments.
 .PP
 .Ss "  Dt_t"
 This is the type of a dictionary handle.
+It contains a field \f5Dt_t.user\fP of type \f5Dtuser_t*\fP (see below).
 .PP
 .Ss "  Dtdisc_t"
 This defines the type of a discipline structure which define the lay-out of
@@ -134,6 +140,22 @@ This is the type of a dictionary object holder (see \f5dtdisc()\fP.)
 .Ss "  Dtstat_t"
 This is the type of a structure to return dictionary statistics (see \f5dtstat()\fP.)
 .PP
+.Ss "  Dtuser_t"
+This is the type of a structure pointed to by \f5Dt_t.user\fP.
+If a discipline function \f5memoryf()\fP was defined, this structure
+will reside in memory allocated via \f5memoryf\fP.
+Although the structure is intended to be used by an application outside of CDT operations,
+the functions \f5dtuserlock()\fP and \f5dtuserdata()\fP
+are provided for certain common usages of the defined fields.
+It should be emphasized, however, that a particular application can choose
+to use this structure in anyway that it sees fit.
+.Cs
+    typedef struct
+    {   unsigned int lock; /* for locking a shared dictionary */
+        Void_t*      data; /* for application-specific data   */
+    } Dtuser_t;
+.Ce
+.PP
 .Ss "DICTIONARY CONTROL"
 .PP
 .Ss "  Dt_t* dtopen(const Dtdisc_t* disc, const Dtmethod_t* meth)"
@@ -183,50 +205,63 @@ In addition, dictionaries on the same view path should
 treat objects in a consistent manner with respect to comparison or hashing.
 If not, undefined behaviors may result.
 .PP
-.Ss "  int dtcustomize(Dt_t* dt, int type, Void_t* arg)"
+.Ss "  int dtcustomize(Dt_t* dt, int type, int action)"
 This customizes a storage method. The \f5type\fP argument
-indicates the type of customization and \f5arg\fP gives additional
+is composed of bits indicating different types of customization.
-information for the operation. Here are the types:
+The \f5action\fP argument, if positive, turns on the desired customization;
+else, turning it off.
+The return value is a bit vector telling the customization types successfully performed.
+
+Here are the types:
 .Tp
 \f5DT_SHARE\fP:
-This turns on/off the share mode for a dictionary.
+This controls the shared or concurrent mode for a dictionary.
-Concurrent accesses of a dictionary not in share mode
+Shared mode allows concurrent threads or processes to safely
-may exhibit undefined behaviors including memory segmentation.
+access objects in a dictionary.
-
+.Tp
-Share mode allows multiple accessors, threads or processes, to access objects.
+\f5DT_ANNOUNCE\fP:
-Such objects could be in the same directory in the case of threads or shared
+This requires each dictionary access operation to invoke
-memory in the case of processes.
+the discipline \f5eventf\fP function to announce an object found or constructed
+by the operation before returning (See the DISCIPLINE section below).
 .Tp
 \f5DT_OPTIMIZE\fP:
 This causes the underlying method to optimize its internal
 data structure. For example, the splay tree underlying \f5Dtoset\fP
 would be balanced.
 .PP
-.Ss "  int dtoptimize(Dt_t* dt)"
+.Ss "  int dtuserlock(Dt_t* dt, unsigned int key, int action)"
-This is a short-hand for invoking \f5dtcustomize()\fP with the \f5DT_OPTIMIZE\fP event.
+This manipulates the lock \f5dt->user->lock\fP.
-.PP
-.Ss "  int dtshare(Dt_t* dt, int type)"
-This turns on or off share mode for dictionary \f5dt\fP depending on whether \f5type\fP
-is positive or non-positive. It returns -1 on failure.
-.PP
-.Ss "  int dtlock(Dt_t* dt, unsigned int key, int type)"
-This globally locks/unlocks a dictionary using the given \f5key\fP.
 It returns 0 on success and -1 on failure.
-The value of \f5key\fP must not be 0.
+The value of \f5key\fP must be non-zero.
-The argument \f5type\fP is used as follows:
+The argument \f5action\fP is used as follows:
 .Tp
-\f5type < 0\fP:
+\f5action < 0\fP:
-Unlock the dictionary if it was locked with \f5key\fP.
+Unlock \f5dt->user.lock\fP if it was locked with \f5key\fP.
-An error will result if the dictionary was locked with a different key.
+An error will result if \f5dt->user->lock\fP was locked with a different key.
 .Tp
-\f5type == 0\fP:
+\f5action == 0\fP:
-Attempt to lock the dictionary with \f5key\fP if it is unlocked.
+Attempt to lock \f5dt->user->lock\fP with \f5key\fP if it is unlocked.
 An error will result if the dictionary was already locked with a different key.
 .Tp
-\f5type > 0\fP:
+\f5action > 0\fP:
-Attempt to lock the dictionary with \f5key\fP.
+Attempt to lock \f5dt->user->lock\fP with \f5key\fP.
-If the dictionary is already locked with a different key,
+If \f5dt->user.lock\fP is already locked with a different key,
-the call will loop and wait until the lock is open to lock it.
+the call will block until \f5dt->user->lock\fP can be locked with the given \f5key\fP.
+
+Note that obtaining or removing a lock with \f5dtuserlock()\fP
+is just a service provided to the
+application for their own use and has nothing to do with dictionary operations
+which may or may not employ their own locking schemes based on the semantics
+of the container data structures in use.
+.PP
+.Ss "  Void_t* dtuserdata(Dt_t* dt, Void_t* data, int set)"
+This function returns the current value of \f5dt->user->data\fP.
+In addition, if \f5set\fP is non-zero,
+the value of \f5dt->user->data\fP will be changed to \f5data\fP.
+.PP
+.Ss "  int dtuserevent(Dt_t* dt, int flags, Void_t* data)"
+This function invokes the discipline event function
+with the event \f5DT_ANNOUNCE|DT_USER|flags\fP and the given data.
 
 .PP
 .Ss "STORAGE METHODS"
@@ -252,13 +287,14 @@ The underlying data structure is a hash table with chaining to handle collisions
 These methods are like \f5Dtset\fP and \f5Dtbag\fP but are based on
 a recursive hashing data structure that allows table extension without
 object relocation. The data structure also supports lock-free
-concurrent search operations for share dictionaries.
+concurrent search operations for shared dictionaries and nearly lock-free
+insertions and deletions.
 .PP
 .Ss "  Dtlist"
 Objects are kept in a list.
 \fIA current object\fP is always defined to be either the head of
 the list or an object resulting from a recent search or insert operation.
-The call \f5dtinsert()\fP will insert a new object
+The calls \f5dtinsert()\fP and \f5dtinstall()\fP will insert a new object
 in front of such a current object
 while the call \f5dtappend()\fP will append in back of it.
 .PP
@@ -313,7 +349,7 @@ i.e., at address \f5(Dtlink_t*)((char*)obj+link)\fP.
 .PP
 .Ss "  Void_t* (*makef)(Dt_t* dt, Void_t* obj, Dtdisc_t* disc)"
 If \f5makef\fP is not \f5NULL\fP,
-\f5dtinsert(dt,obj)\fP or \f5dtappend()\fP will call it
+\f5dtinsert()\fP, \f5dtinstall()\fP or \f5dtappend()\fP will call it
 to make a copy of \f5obj\fP suitable for insertion into \f5dt\fP.
 If \f5makef\fP is \f5NULL\fP, \f5obj\fP itself will be inserted into \f5dt\fP.
 .PP
@@ -362,11 +398,11 @@ On a negative return value, \f5dtopen()\fP will return failure.
 
 On a zero return value, \f5eventf()\fP may set \f5*(Void_t**)data\fP to some non-\f5NULL\fP
 value to indicate that the dictionary structure itself should be allocated
-along with the \f5Dtdisc_t.data\fP section.
+along with the \f5Dt_t.data\fP section.
 Otherwise, it will be allocated separately with \f5malloc(3)\fP.
 
 On a positive return value, the dictionary is being reconstructed
-based on existing states of some previous dictionary.
+based on the existing states of some previous dictionary.
 In this case, \f5eventf()\fP should set \f5*(Void_t**)data\fP to point to
 the field \f5Dt_t.data\fP of the corresponding previous dictionary (see \f5DT_CLOSE\fP below).
 If the handle of the previous dictionary was created as discussed above
@@ -386,7 +422,7 @@ The return value of \f5eventf\fP is significant as follows:
 On a negative return value, \f5dtclose()\fP will return failure.
 
 On a zero return value, all dictionary objects will be deleted and
-and all associated memory will be freed.
+and associated memory freed.
 
 On a positive return value, allocated objects and memory will be kept intact.
 This means that \f5dt->data\fP remains intact and can be reused in some future
@@ -398,31 +434,60 @@ This event is raised at the end of the process to close a dictionary.
 The return value of \f5eventf()\fP will be ignored.
 .Tp
 \f5DT_DISC\fP:
-The discipline of \f5dt\fP is being changed to a new one given in
+This event indicates that the discipline of \f5dt\fP is being changed to a new one given in
 \f5(Dtdisc_t*)data\fP.
 .Tp
 \f5DT_METH\fP:
-The method of \f5dt\fP is being changed to a new one given in
+This event indicates that the method of \f5dt\fP is being changed to a new one given in
 \f5(Dtmethod_t*)data\fP.
 .Tp
 \f5DT_HASHSIZE\fP:
-This event is applicable to
+This event is raised by the methods \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP
-the methods \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP.
+to ask an application to suggest a size (measured in objects) for the data structure in use.
-It is typically issued when the respective internal data structure of
+This is useful, for example, to set a initial size for a hash table to reduce collisions and rehashing.
-a method is about to be initialized.
+On each call, \f5*(ssize_t*)data\fP will initially have the current size
-If the return value of the event handling function is positive,
+(which should be \f50\fP on the first call).
-\f5*(ssize_t*)data\fP is examined for further action;
+
-else, it is ignored.
+The return value of the event handling function indicates actions to be taken.
-A positive return value means that the event function wishes to suggest a table size.
+If non-positive, the method will proceed with its default actions.
-It does that by setting \f5*(ssize_t*)data\fP to the desired size.
+Otherwise, the application may set \f5*(ssize_t*)data\fP to suggest a table size.
-Then, the actual table size will be the maximum of the absolute value
+The actual table size will be based on the absolute value of \f5*(ssize_t*)data\fP
-of \f5*(ssize_t*)data\fP and some predefined value set by the method.
+but may be modified to suit for the data structure in use.
-In addition, if \f5*(ssize_t*)data\fP was negative,
+Further, if \f5*(ssize_t*)data\fP was negative, the size of the hash table will be fixed going forward.
-the \f5Dtset\fP and \f5Dtbag\fP methods will never resize the hash table.
 .Tp
 \f5DT_ERROR\fP:
-This event announces an error that occurred during some operations.
+This event states an error that occurred during some operations, e.g.,
-The argument \f5(char*)data\fP is a null-terminated string describing the error.
+\f5dtinsert()\fP or \f5dtinstall()\fP failing to create a new object due to a memory allocation error.
+The argument \f5(char*)data\fP is a null-terminated string describing the problem.
+.Tp
+\f5DT_ANNOUNCE\fP:
+The event will be a combination of this bit and a bit indicating a successful operation.
+For example, \f5DT_ANNOUNCE|DT_SEARCH\fP announces that \f5dtsearch()\fP
+found the object that was searched for.  The \f5data\fP argument points to the object itself.
+
+The bits representing operations that can cause an announcement are:
+\f5DT_INSERT\fP,
+\f5DT_DELETE\fP,
+\f5DT_REMOVE\fP,
+\f5DT_SEARCH\fP,
+\f5DT_NEXT\fP,
+\f5DT_PREV\fP,
+\f5DT_FIRST\fP,
+\f5DT_LAST\fP,
+\f5DT_MATCH\fP,
+\f5DT_ATTACH\fP,
+\f5DT_DETACH\fP,
+\f5DT_APPEND\fP,
+\f5DT_INSTALL\fP,
+\f5DT_LEAST\fP, and
+\f5DT_MOST\fP.
+
+Note that a call to \f5dtinsert()\fP or \f5dtattach()\fP may return
+a successfully inserted new object or a found matching object.
+For \f5dtinsert()\fP, the former case will be announced as \f5DT_ANNOUNCE|DT_INSERT\fP while
+the latter as \f5DT_ANNOUNCE|DT_INSERT|DT_SEARCH\fP.
+For \f5dtattach()\fP, the events will be similarly announced as \f5DT_ANNOUNCE|DT_ATTACH\fP
+and \f5DT_ANNOUNCE|DT_ATTACH|DT_SEARCH\fP.
 .PP
 .Ss "#define DTOFFSET(struct_s,member)"
 This macro function computes the offset of \f5member\fP from the start
@@ -436,25 +501,30 @@ with the given values.
 .Ss "OBJECT OPERATIONS"
 .PP
 .Ss "  Void_t* dtinsert(Dt_t* dt, Void_t* obj)"
+.Ss "  Void_t* dtinstall(Dt_t* dt, Void_t* obj)"
 .Ss "  Void_t* dtappend(Dt_t* dt, Void_t* obj)"
 These functions add an object prototyped by \f5obj\fP into \f5dt\fP.
+See \f5Dtdisc_t.makef\fP for object construction.
 \f5dtinsert()\fP and \f5dtappend()\fP perform the same function
 for all methods except for \f5Dtlist\fP (see \f5Dtlist\fP for details).
-If there is an existing object in \f5dt\fP matching \f5obj\fP
+For \f5Dtset\fP, \f5Dtrhset\fP or \f5Dtoset\fP,
-and the storage method is \f5Dtset\fP, \f5Dtrhset\fP or \f5Dtoset\fP,
+if there is an object in \f5dt\fP matching \f5obj\fP
-\f5dtinsert()\fP and \f5dtappend()\fP will simply return the matching object.
+\f5dtinsert()\fP and \f5dtappend()\fP will not insert a new object.
-Otherwise, a new object is inserted according to the method in use.
+On the other hand, \f5dtinstall()\fP remove such a matching
-See \f5Dtdisc_t.makef\fP for object construction.
+object then insert the new object.
-The new object or a matching object as noted will be returned on success
+
-while \f5NULL\fP is returned on error.
+On failure, \f5dtinsert()\fP and \f5dtinstall()\fP return \f5NULL\fP.
+Otherwise, the return value is either the newly inserted object
+or the matching object as noted.
 .PP
 .Ss "  Void_t* dtdelete(Dt_t* dt, Void_t* obj)"
-If \f5obj\fP is \f5NULL\fP, methods \f5Dtstack\fP and \f5Dtqueue\fP
+.Ss "  Void_t* dtremove(Dt_t* dt, Void_t* obj)"
-delete respectively stack top or queue head while other methods do nothing.
+When \f5obj\fP is not \f5NULL\fP, \f5dtdelete()\fP removes some object \fImatching\fP \f5obj\fP
-If \f5obj\fP is not \f5NULL\fP, an object matching \f5obj\fP is deleted.
+while \f5dtremove()\fP removes \f5obj\fP itself if it exists.
+When \f5obj\fP is \f5NULL\fP, if the method is \f5Dtstack\fP or \f5Dtqueue\fP
+then the stack top or queue head is respectively deleted.
 See \f5Dtdisc_t.freef\fP for object destruction.
-\f5dtdelete()\fP returns the deleted object (even if it was deallocated)
+\f5dtdelete()\fP and \f5dtremove()\fP return the deleted object or \f5NULL\fP.
-or \f5NULL\fP on error.
 .PP
 .Ss "  Void_t* dtattach(Dt_t* dt, Void_t* obj)"
 This function is similar to \f5dtinsert()\fP but \f5obj\fP itself
@@ -469,13 +539,21 @@ from \f5dt\fP will not be freed (via the discipline \f5freef\fP function).
 .Ss "  Void_t* dtmatch(Dt_t* dt, Void_t* key)"
 These functions find an object matching \f5obj\fP or \f5key\fP either from \f5dt\fP or
 from some dictionary accessible from \f5dt\fP via a viewpath (see \f5dtview()\fP.)
-\f5dtsearch()\fP and \f5dtmatch()\fP return the matching object or
+The return value is the matching object or \f5NULL\fP.
-\f5NULL\fP on failure.
 .PP
 .Ss "  Void_t* dtfirst(Dt_t* dt)"
 .Ss "  Void_t* dtnext(Dt_t* dt, Void_t* obj)"
-\f5dtfirst()\fP returns the first object in \f5dt\fP.
+.Ss "  Void_t* dtlast(Dt_t* dt)"
-\f5dtnext()\fP returns the object that follows an object matching \f5obj\fP.
+.Ss "  Void_t* dtprev(Dt_t* dt, Void_t* obj)"
+These functions assume some object ordering (more below) and can be used
+to iterate over all objects.
+\f5dtfirst()\fP returns the first object in \f5dt\fP or \f5NULL\fP if the
+dictionary is empty.
+\f5dtnext()\fP returns the object coming after \f5obj\fP
+or \f5NULL\fP if there is no such object.
+\f5dtlast()\fP and \f5dtprev()\fP are like \f5dtfirst()\fP and \f5dtnext()\fP
+but work in reverse order.
+
 Objects are ordered based on the storage method in use.
 For \f5Dtoset\fP and \f5Dtobag\fP, objects are ordered by object comparisons.
 For \f5Dtstack\fP, objects are ordered in reverse order of insertion.
@@ -484,33 +562,47 @@ For \f5Dtlist\fP, objects are ordered by list position.
 For \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP,
 objects are ordered by some internal order defined at the time when these
 functions are called.
+In fact, both forward and reverse orders are defined to be the same
+for these methods.
+
+Objects in a dictionary or a viewpath of dictionaries can be walked using
+\f5for(;;)\fP loops as below:
 
-Objects in a dictionary or a viewpath can be walked using
-a \f5for(;;)\fP loop as below.
 .Cs
     for(obj = dtfirst(dt); obj; obj = dtnext(dt,obj))
 .Ce
+or
+.Cs
+    for(obj = dtlast(dt); obj; obj = dtprev(dt,obj))
+.Ce
+
+The argument \f5obj\fP of \f5dtnext()\fP or \f5dtprev()\fP is treated specially
+for a method that allows multiple equal elements such as \f5Dtobag\fP or \f5Dtbag\fP.
+If it is in the dictionary, then the returned object will be respectively
+immediately before or after it in the implicitly defined object ordering.
+If it is not in the dictionary but still matching a group of objects,
+then the returned object will be immediately after the last or before the first
+of the group respectively.
 .PP
-.Ss "  Void_t* dtlast(Dt_t* dt)"
-.Ss "  Void_t* dtprev(Dt_t* dt, Void_t* obj)"
-\f5dtlast()\fP and \f5dtprev()\fP are like \f5dtfirst()\fP and \f5dtnext()\fP
-but work in reverse order.
-For \f5Dtset\fP, \f5Dtbag\fP, \f5Dtrhset\fP and \f5Dtrhbag\fP,
-both reverse and forward orders are the same.
-Note that dictionaries on a viewpath are still walked in the order
-of the viewpath.
 .PP
-.Ss "  Void_t* dtleast(Dt_t* dt, Void_t* obj)"
+.Ss "  Void_t* dtatleast(Dt_t* dt, Void_t* obj)"
-.Ss "  Void_t* dtmost(Dt_t* dt, Void_t* obj)"
+.Ss "  Void_t* dtatmost(Dt_t* dt, Void_t* obj)"
-\f5dtleast()\fP returns the smallest object greater or equal to \f5obj\fP.
+\f5dtatleast()\fP returns the smallest object greater or equal to \f5obj\fP.
-\f5dtmost()\fP returns the largest object smaller or equal to \f5obj\fP.
+\f5dtatmost()\fP returns the largest object smaller or equal to \f5obj\fP.
+In addition, if there are multiple such objects in \f5dt\fP
+(i.e., when a bag method was used), then
+\f5dtatmost()\fP returns the first instance of such an object while
+\f5dtatleast()\fP returns the last one.
+Both functions return \f5NULL\fP if the desired object does not exist.
+
 Again, object ordering depends on the storage method in use.
-For example, with \f5Dtoset\fP and \f5Dtobag\fP, the ordering of objects
+With \f5Dtoset\fP and \f5Dtobag\fP, objects are linearly ordered by
-is well-defined and it is possible to call \f5dtleast()\fP or \f5dtmost()\fP
+the discipline comparison function.
+As such, it is possible to call \f5dtatleast()\fP or \f5dtatmost()\fP
 on an object not in the dictionary and still get a meaningful result.
-On the other hand, with \f5Dtset\fP or \f5Dtrhset\fP, such a call will
+Storage methods other than \f5Dtoset\fP and \f5Dtobag\fP do not have
-essentially be the same as \f5dtsearch()\fP because without matching
+an explicit ordering so \f5dtatmost()\fP
-an object, it cannot be determined what comes before or after.
+and \f5dtatleast()\fP will return \f5NULL\fP when there are no matching objects.
 .PP
 .Ss "  dtwalk(Dt_t* dt, int (*userf)(Dt_t*, Void_t*, Void_t*), Void_t* data)"
 This function calls \f5(*userf)(walk,obj,data)\fP on each object in \f5dt\fP and
@@ -528,6 +620,7 @@ to walk a single dictionary can incur significant cost due to function calls.
 For efficient walking of a single directory (i.e., no viewpathing),
 \f5dtflatten()\fP and \f5dtlink()\fP can be used.
 Objects in \f5dt\fP are made into a linked list and walked as follows:
+
 .Cs
     for(link = dtflatten(dt); link; link = dtlink(dt,link) )
 .Ce
@@ -537,11 +630,48 @@ not \f5Void_t*\fP. That is, it returns a dictionary holder pointer,
 not a user object pointer
 (although both are the same if the discipline field \f5link\fP is zero.)
 The macro function \f5dtlink()\fP
-returns the dictionary holder object following \f5link\fP.
+returns the dictionary holder object following \f5link\fP and
-The macro function \f5dtobj(dt,link)\fP
+the macro function \f5dtobj(dt,link)\fP
 returns the user object associated with \f5link\fP,
-Beware that the flattened object list is unflattened on any
+Beware that a flattened object list is not guaranteed to maintain integrity
-dictionary operations other than \f5dtlink()\fP.
+if any dictionary operation other than \f5dtlink()\fP is performed
+(for example, this is important to watch out for
+if a dictionary is in \f5DT_SHARE\fP mode).
+.PP
+.Ss "  Void_t* dtstart(Dt_t* dt, Void_t* obj);"
+This function starts a path for walking a dictionary.
+Note that such a path is restricted to \f5dt\fP only while disregarding
+all viewpath dictionaries (see \f5dtview()\fP).
+On success, a structure
+to be used in \f5dtstep()\fP for walking the path is returned.
+Otherwise, \f5NULL\fP is returned.
+
+If \f5obj\fP is \f5NULL\fP, the path starts at the same object returned by \f5dtfirst()\fP.
+If \f5obj\fP is not \f5NULL\fP, it must match some object in the dictionary \f5dt\fP
+and the path will start there. No matching object will result in error.
+.PP
+.Ss "  Void_t* dtstop(Dt_t* dt, Void_t* path);"
+This function ends a path and releases all memory source associated with it.
+.PP
+.Ss "  Void_t* dtstep(Dt_t* dt, Void_t* path);"
+This function returns the object at current position in the given \f5path\fP.
+Successive calls move forward one object at a time in the same order that \f5dtnext()\fP
+does in the example \f5for(;;)\fP loop above. If there is no more object in the path,
+\f5dtstep()\fP returns \f5NULL\fP.
+
+Below is a code fragment showing how to create and walk a path of objects.
+This object walking method is more restricted than the \f5dtfirst()/dtnext()\fP method
+since viewpathed dictionaries are ignored.
+However, it allows multiple paths to be traversed concurrently in the
+most efficient manner possible as supported by the underlying data structures.
+.Cs
+    path = dtstart(dt, firstobj);
+    for(obj = dtstep(dt, path); obj; obj = dtstep(dt,path))
+    {
+        ...
+    }
+    dtstop(dt, path);
+.Ce
 .PP
 .Ss "  Dtlink_t* dtextract(Dt_t* dt)"
 .Ss "  Dtlink_t* dtrestore(Dt_t* dt, Dtlink_t* list)"
@@ -579,39 +709,232 @@ This has the number of objects in the dictionary.
 .Tp
 \f5ssize_t mlev\fP:
 This returns the maximum number of levels in the data structure used for object storage, i.e.,
-the binary tree or the recursive hash table.
+the binary tree (e.g., \f5Dtoset\fP) or the recursive hash table based on a trie structure (e.g., \f5Dtrhset\fP).
-For a hash table with chaining (i.e., \f5Dtset\fP and \f5Dtbag\fP),
+For a hash table with chaining (e.g., \f5Dtset\fP and \f5Dtbag\fP),
 it gives the length of the longest chain.
 .Tp
 \f5ssize_t lsize[]\fP:
 This gives the object counts at each level.
-For a hash table with chaining (i.e., \f5Dtset\fP and \f5Dtbag\fP),
+For a hash table with chaining (e.g., \f5Dtset\fP and \f5Dtbag\fP),
 a level is defined as objects at that position in their chains.
-Since chains can be arbitrarily long, the report is limited
+The reported levels is limited to less than \f5DT_MAXSIZE\fP.
-to objects at a level less than \f5DT_MAXSIZE\fP.
 .Tp
 \f5ssize_t tsize[]\fP:
-For a hash table using a trie structure, this counts the number of
+For a recursive hash table using a trie structure (\f5Dtrehash\fP), this counts the number of
 sub-tables at each level. For example, \f5tsize[0]\fP should be 1
 only for this hash table type.
+The reported levels is limited to less than \f5DT_MAXSIZE\fP.
+.Tp
+\f5char* mesg\fP:
+A summary message of some of the statistics.
 .PP
 .Ss "HASH FUNCTIONS"
 .PP
-.Ss "  unsigned int dtcharhash(unsigned int h, char c)"
 .Ss "  unsigned int dtstrhash(unsigned int h, char* str, int n)"
-These functions compute hash values from bytes or strings.
+This function computes a new hash value from string \f5str\fP and seed value \f5h\fP.
-\f5dtcharhash()\fP computes a new hash value from byte \f5c\fP and seed value \f5h\fP.
-\f5dtstrhash()\fP computes a new hash value from string \f5str\fP and seed value \f5h\fP.
 If \f5n\fP is positive, \f5str\fP is a byte array of length \f5n\fP;
 otherwise, \f5str\fP is a null-terminated string.
 .PP
+.SH CONCURRENCY PROGRAMMING NOTES
+Applications requiring concurrent accesses of a dictionary whether via separate threads
+or processes using shared memory should turn on shared mode for the dictionary.
+CDT uses locking and lockless data structures to
+provide safe concurrent accesses of objects.
+Much of this work is based on the atomic scalar operations available in \fIlibaso(3)\fP.
+
+Even though CDT only considers objects
+via the attributes specified in a discipline structure,
+practical objects will often have many more attributes germane to the needs of an application.
+Thus, beyond safe concurrent dictionary operations, an application must also
+protect objects in concurrent computations outside of CDT.
+In particular, both \fIobject deletion\fP and \fIobject creation\fP should be handled with care.
+
+The deletion case is relatively simple.
+No object should be destroyed as long as there is a reference to it.
+This guarantee is automatic when some garbage collection scheme is in place.
+Otherwise, some form of reference counting could be used to make sure
+that only objects with no reference would be deleted.
+An example to be given below discusses how reference counting could be
+done using the \f5DT_ANNOUNCE\fP feature of CDT to ensure correct timing
+for object deletion.
+
+In general, object attributes should be well-defined before they are used.
+The simplest way to ensure this is to completely construct an object before
+before inserting it into a shared dictionary.
+However, an application using complex objects may try
+to avoid unnecessary construction work as follows.
+First, only a partial object with minimal information needed for dictionary operations
+is constructed.
+Then, either\f5dtinsert()\fP or \f5dtattach()\fP is called to insert this partial object
+into the dictionary. If the call returns this same object, then it was properly inserted and
+the rest of its attributes could then be filled in.
+If only a matching object is returned, then the new object is simply discarded.
+Although this object construction strategy works well in single-threaded code,
+it can cause references to uninitialized data in concurrent computations
+because objects are accessible by concurrent code
+as soon as \f5dtinsert()\fP or \f5dtattach()\fP returns.
+A way to solve this problem is to make sure that an incomplete object
+is completed before allowing any dictionary operation accessing such an object
+to return it to the application.
+
+Both reference counting for safe objection deletion and ensuring readiness
+on object creation can be coordinate with CDT via the event \f5DT_ANNOUNCE\fP.
+An example of how to do this is given next.
+Objects are assumed to be of type \f5Obj_t\fP and have two
+fields: \f5ready\fP to indicate the readiness of an object
+and \f5refn\fP for reference counting.
+Both fields \f5ready\fP and \f5refn\fP are initialized to zero.
+Below are the relevant discipline functions \f5Dtdisc_t.eventf\fP
+and \f5Dtdisc_t.freef\fP to handle events and to free an object:
+
+.Cs
+    int eventf(Dt_t* dt, int type, Void_t* arg, Dtdisc_t* disc)
+    {
+        if(type & DT_ANNOUNCE)
+        {
+            if(!(type & DT_DELETE) )
+            {
+                Obj_t  *obj = (Obj_t*)arg;
+
+                if(type & ~(DT_ANNOUNCE|DT_INSERT|DT_ATTACH))
+                    while(asogetchar(&obj->ready) == 0 )
+                        asorelax(1);
+
+                asoaddint(&obj->refn, 1);
+            }
+
+            return 0;
+        }
+        ...
+    }
+
+    void freef(Dt_t* dt, Void_t* arg, Dtdisc_t* disc)
+    {
+        Obj_t  *obj = (Obj_t*)arg;
+
+        while(asogetchar(&obj->ready) == 0 )
+            asorelax(1);
+
+        while(asogetint(&obj->refn) > 0 )
+            asorelax(1);
+
+        ... destroy the object ...
+    }
+.Ce
+
+Recall that each operation announcement is composed of \f5DT_ANNOUNCE\fP
+and some bits to indicate the operation itself.
+The test to exclude \f5dtdelete()\fP (indicated by the bit \f5DT_DELETE\fP)
+in \f5eventf()\fP is needed because an announcement always occurs
+right before the relevant
+CDT operation returns and, in the case of \f5dtdelete()\fP,
+the object may/will be already destroyed at that time.
+
+The \f5while()\fP loops in both \f5eventf()\fP and \f5freef()\fP cause
+the relevant operations to wait until the object is \fIready\fP (i.e.,
+all of its attributes are constructed) before proceeding.
+The \f5asorelax(1)\fP call yields control of the processor for 1 nanosecond
+so other processes can do their work.
+Note that the test for \f5~(DT_ANNOUNCE|DT_INSERT|DT_ATTACH)\fP in \f5eventf()\fP
+means that the loop will execute for all CDT operations except for
+the \f5dtinsert()\fP or \f5dtattach()\fP call that actually inserts \f5obj\fP
+into the dictionary (more on this below).
+
+When the \f5while\fP loop finished, the construction of object \f5obj\fP is known
+to be completed. \f5eventf()\fP increases the reference count \f5obj->refn\fP by one
+before the respective operation returns \f5obj\fP to the calling code.
+On the other hand, \f5freef()\fP waits for the reference
+count to reach zero before proceeding to destroy the object.
+Waiting for object readiness in \f5freef()\fP before object destruction is necessary
+to avoid any issues with deleting uninitialized data.
+Again, it should be emphasized that reference counting
+is needed only for a memory management model where objects can be freed
+regardless of whether or not there are any references to them.
+Applications that use some form of garbage collection in general or
+for dictionary objects may ignore doing reference counting as done in this example.
+
+Next, consider a fragment of code to access
+objects concurrently from different threads or processes:
+
+.Cs
+    if((obj = dtmatch(dt, "key_string")) != NULL)
+    {
+         ...process the object obj...
+
+         asosubint(&obj->refn, 1);
+         dtdelete(dt, obj);
+    }
+.Ce
+
+The sequence of activities is as follows.
+First, the call \f5dtmatch()\fP retrieves an object \f5obj\fP.
+An announcement would be made during the call just before \f5obj\fP is returned
+causing the reference count of \f5obj\fP to be increased by one.
+After processing \f5obj\fP, the reference count is decreased by one using the
+atomic subtraction operator \f5asosubint()\fP.
+Then, \f5dtdelete()\fP is called to delete the object.
+
+A possible danger is that concurrent calls to \f5dtdelete()\fP
+may end up causing the same memory to be freed more than once.
+Fortunately, this cannot happen.
+CDT guarantees that, of all the concurrent calls to \f5dtdelete()\fP on \f5obj\fP,
+only one will get far enough to make the \f5freef()\fP call while others do nothing.
+
+Finally, consider a code fragment to construct and use the object \f5obj\fP:
+
+.Cs
+    ... construct a partial object obj ...
+    if((insobj = dtinsert(dt, obj)) == obj )
+    {
+        ... fully construct obj ...
+        asocaschar(&obj->ready, 0, 1);
+
+        ... compute based on obj...
+        asosubint(&obj->refn, 1);
+    }
+    else
+    {   ... destroy the partial obj ...
+
+        ... compute based on insobj...
+        asosubint(&insobj->refn, 1);
+    }
+.Ce
+
+After the \f5dtinsert()\fP call returns,
+all other concurrent computations invoking dictionary operations to access \f5obj\fP
+will be blocked in the \f5eventf()\fP function until \f5obj->ready\fP is set to 1
+by the above \f5asocaschar()\fP call.
+As this is a concurrent computing application,
+the above code fragment itself can be
+executed in parallel with different but equivalent versions of \f5obj\fP.
+In that case, only one \f5dtinsert()\fP call will succeed in inserting a new object
+while the others will report a matching object, i.e., the one actually inserted.
+The announcement of the successful case is \f5DT_ANNOUNCE|DT_INSERT\fP
+while the announcement of the other cases is \f5DT_ANNOUNCE|DT_INSERT|DT_SEARCH\fP.
+The bit \f5DT_SEARCH\fP causes \f5eventf()\fP to
+to run the loop waiting for object completion. Thus, overall, except for the single case
+of a successful insertion of a new object, all other dictionary accesses that involve
+this object will return only when the object is ready.
+
+Note that, for simplicity, the possibility of failure was ignored in the example.
+In both successful outcomes of \f5dtinsert()\fP, the reference count of an
+appropriate object will be increased by one. Thus, care must be taken to
+reduce that reference count for the object after it is no longer needed.
+Else, per this example implementation, a deletion of such an object will
+cause an infinite loop in the discipline \f5freef()\fP function.
+It is possible to implement a delayed object destruction scheme
+that avoids an infinite loop waiting for the reference count to drop to zero.
+However, a discussion of that is beyond the scope of this document.
+.PP
 .SH IMPLEMENTATION NOTES
-\f5Dtlist\fP, \f5Dtstack\fP and \f5Dtqueue\fP are based on doubly linked list.
+\f5Dtlist\fP, \f5Dtstack\fP, \f5Dtdeque\fP and \f5Dtqueue\fP are based on doubly linked list.
 \f5Dtoset\fP and \f5Dtobag\fP are based on top-down splay trees.
-\f5Dtset\fP and \f5Dtbag\fP are based on hash tables with
+\f5Dtset\fP and \f5Dtbag\fP are based on hash tables with collision chains.
-move-to-front collision chains.
 \f5Dtrhset\fP and \f5Dtrhbag\fP are based on a recursive hashing data structure
 that avoids table resizing.
 .PP
+.SH SEE ALSO
+libaso(3), libvmalloc(3)
+.PP
 .SH AUTHOR
 Kiem-Phong Vo, kpv@research.att.com