Fix enum type definition pre-parsing for shcomp and dot/source

Parser limitations prevent shcomp or source from handling enum
types correctly:

    $ cat /tmp/colors.sh
    enum Color_t=(red green blue orange yellow)
    Color_t -A Colors=([foo]=red)

    $ shcomp /tmp/colors.sh > /dev/null
    /tmp/colors.sh: syntax error at line 2: `(' unexpected
    $ source /tmp/colors.sh
    /bin/ksh: source: syntax error: `(' unexpected

Yet, for types created using 'typeset -T', this works. This is done
via a check_typedef() function that preliminarily adds the special
declaration builtin at parse time, with details to be filled in
later at execution time.

This hack will produce ugly undefined behaviour if the definition
command creating that type built-in is then not actually run at
execution time before the type built-in is accessed.

But the hack is necessary because we're dealing with a fundamental
design flaw in the ksh language. Dynamically addable built-ins that
change the syntactic parsing of the shell language on the fly are
an absurdity that violates the separation between parsing and
execution, which muddies the waters and creates the need for some
kind of ugly hack to keep things like shcomp more or less working.

This commit extends that hack to support enum.

src/cmd/ksh93/sh/parse.c:
- check_typedef():
  - Add 'intypeset' parameter that should be set to 1 for typeset
    and friends, 2 for enum.
  - When processing enum arguments, use AST getopt(3) to skip over
    enum's options to find the name of the type to be defined.
    (getopt failed if we were running a -c script; deal with this
    by zeroing opt_info.index first.)
- item(): Update check_typedef() call, passing lexp->intypeset.
- simple(): Set lexp->intypeset to 2 when processing enum.

The rest of the changes are all to support the above and should be
fairly obvious, except:

src/cmd/ksh93/bltins/enum.c:
- enuminfo(): Return on null pointer, avoiding a crash upon
  executing 'Type_t --man' if Type_t has not been fully defined due
  to the definition being pre-added at parse time but not executed.
  It's all still wrong, but a crash is worse.

Resolves: https://github.com/ksh93/ksh/issues/256

NEWS

@@ -3,6 +3,12 @@ For full details, see the git log at: https://github.com/ksh93/ksh
 
 Any uppercase BUG_* names are modernish shell bug IDs.
 
+2021-11-21:
+
+- It is now possible to use types defined by 'enum' in contexts where the
+  script is entirely parsed before (or without) being executed, such as
+  dotted/sourced scripts and scripts compiled by shcomp.
+
 2021-11-20:
 
 - Listing types with 'typeset -T' no longer displays incomplete versions of

src/cmd/ksh93/bltins/enum.c

@@ -21,8 +21,8 @@
 #pragma prototyped
 #include	"defs.h"
 
-static const char enum_usage[] =
-"[-?@(#)$Id: enum (AT&T Research) 2008-01-08 $\n]"
+const char sh_optenum[] =
+"[-?@(#)$Id: enum (ksh 93u+m) 2021-11-21 $\n]"
 "[--catalog?" ERROR_CATALOG "]"
 "[+NAME?enum - create an enumeration type]"
 "[+DESCRIPTION?\benum\b is a declaration command that creates an enumeration "
@@ -43,7 +43,7 @@ static const char enum_usage[] =
 ;
 
 static const char enum_type[] =
-"[-1c?\n@(#)$Id: type (AT&T Labs Research) 2008-01-08 $\n]"
+"[-1c?\n@(#)$Id: type (ksh 93u+m) 2021-11-21 $\n]"
 "[--catalog?" ERROR_CATALOG "]"
 "[+NAME?\f?\f - create an instance of type \b\f?\f\b]"
 "[+DESCRIPTION?The \b\f?\f\b declaration command creates a variable for "
@@ -98,6 +98,8 @@ static int enuminfo(Opt_t* op, Sfio_t *out, const char *str, Optdisc_t *fp)
 	const char	*v;
 	np = *(Namval_t**)(fp+1);
 	ep = (struct Enum*)np->nvfun;
+	if(!ep)
+		return(0);
 	if(strcmp(str,"default")==0)
 		sfprintf(out,"\b%s\b",ep->values[0]);
 	else if(strcmp(str,"case")==0)
@@ -192,7 +194,7 @@ int b_enum(int argc, char** argv, Shbltin_t *context)
 	cmdinit(argc, argv, context, ERROR_CATALOG, ERROR_NOTIFY);
 	for (;;)
 	{
-		switch (optget(argv, enum_usage))
+		switch (optget(argv, sh_optenum))
 		{
 		case 'i':
 			iflag = 'i';

src/cmd/ksh93/data/builtins.c

@@ -81,12 +81,12 @@ const struct shtable3 shtab_builtins[] =
 	".",		NV_BLTIN|BLT_ENV|BLT_SPC,	bltin(dot_cmd),
 	"source",	NV_BLTIN|BLT_ENV,		bltin(dot_cmd),
 	"return",	NV_BLTIN|BLT_ENV|BLT_SPC,	bltin(return),
+	"enum",		NV_BLTIN|BLT_ENV|BLT_DCL,	bltin(enum),
 /*
  * Builtins without offset macros in include/builtins.h follow.
  */
 	"alias",	NV_BLTIN|BLT_ENV,		bltin(alias),
 	"hash",		NV_BLTIN|BLT_ENV,		bltin(alias),
-	"enum",		NV_BLTIN|BLT_ENV|BLT_DCL,	bltin(enum),
 	"eval",		NV_BLTIN|BLT_ENV|BLT_SPC|BLT_EXIT,bltin(eval),
 	"exit",		NV_BLTIN|BLT_ENV|BLT_SPC,	bltin(return),
 	"fc",		NV_BLTIN|BLT_ENV|BLT_EXIT,	bltin(hist),

src/cmd/ksh93/include/builtins.h

@@ -60,6 +60,7 @@
 #define SYSDOT		(shgd->bltin_cmds+20)	/* . */
 #define SYSSOURCE	(shgd->bltin_cmds+21)	/* source */
 #define SYSRETURN	(shgd->bltin_cmds+22)	/* return */
+#define SYSENUM		(shgd->bltin_cmds+23)	/* enum */
 
 /* entry point for shell special builtins */
 
@@ -168,6 +169,7 @@ extern const char sh_optdot[];
 #ifndef ECHOPRINT
     extern const char sh_optecho[];
 #endif /* !ECHOPRINT */
+extern const char sh_optenum[];
 extern const char sh_opteval[];
 extern const char sh_optexec[];
 extern const char sh_optredirect[];

src/cmd/ksh93/include/shlex.h

@@ -45,7 +45,7 @@ typedef struct  _shlex_
 	char		aliasok;	/* on when alias is legal */
 	char		assignok;	/* on when name=value is legal */
 	char		inexec;		/* on when processing exec */
-	char		intypeset;	/* on when processing typeset */
+	char		intypeset;	/* 1 when processing typeset, 2 when processing enum */
 	char		comp_assign;	/* in compound assignment */
 	char		comsub;		/* parsing command substitution */
 	char		noreserv;	/* reserved works not legal */

src/cmd/ksh93/include/version.h

@@ -21,7 +21,7 @@
 
 #define SH_RELEASE_FORK	"93u+m"		/* only change if you develop a new ksh93 fork */
 #define SH_RELEASE_SVER	"1.0.0-beta.2"	/* semantic version number: https://semver.org */
-#define SH_RELEASE_DATE	"2021-11-20"	/* must be in this format for $((.sh.version)) */
+#define SH_RELEASE_DATE	"2021-11-21"	/* must be in this format for $((.sh.version)) */
 #define SH_RELEASE_CPYR	"(c) 2020-2021 Contributors to ksh " SH_RELEASE_FORK
 
 /* Scripts sometimes field-split ${.sh.version}, so don't change amount of whitespace. */

src/cmd/ksh93/sh.1

@@ -8679,3 +8679,7 @@ won't be executed until the foreground job terminates.
 It is a good idea to leave a space after the comma operator in
 arithmetic expressions to prevent the comma from being interpreted
 as the decimal point character in certain locales.
+.PP
+Commands that add type definitions (\f3enum\fP, \f3typeset -T\fP)
+must be run unconditionally and in the main shell environment.
+Defining types conditionally or in a subshell will cause undefined behavior.

src/cmd/ksh93/sh/parse.c

@@ -183,11 +183,19 @@ static void typeset_order(const char *str,int line)
 }
 
 /*
- * add type definitions when compiling with -n
+ * Pre-add type declaration built-ins at parse time to avoid
+ * syntax errors when using -c, shcomp, '.'/source or eval.
+ *
+ * This hack has a bad side effect: defining a type with 'typeset -T' or 'enum'
+ * in a subshell or an 'if false' block will cause an inconsistent state. But
+ * as these built-ins alter the syntax of the shell, it's necessary for making
+ * them work if we're parsing an entire script before or without executing it.
+ *
+ * intypeset==1 for typeset & friends; intypeset==2 for enum
  */
-static void check_typedef(struct comnod *tp)
+static void check_typedef(struct comnod *tp, char intypeset)
 {
-	char	*cp=0;
+	char	*cp=0;		/* name of built-in to pre-add */
 	if(tp->comtyp&COMSCAN)
 	{
 		struct argnod *ap = tp->comarg;
@@ -211,8 +219,18 @@ static void check_typedef(struct comnod *tp)
 	else
 	{
 		struct dolnod *dp = (struct dolnod*)tp->comarg;
-		char **argv = dp->dolval + dp->dolbot+1;
-		while((cp= *argv++) && memcmp(cp,"--",2))
+		char **argv = dp->dolval + ARG_SPARE;
+		if(intypeset==2)
+		{
+			/* Skip over 'enum' options */
+			opt_info.index = 0;
+			while(optget(argv, sh_optenum))
+				;
+			if(error_info.errors)
+				return;
+			cp = argv[opt_info.index];
+		}
+		else while((cp = *argv++) && memcmp(cp,"--",2))
 		{
 			if(sh_isoption(SH_NOEXEC))
 				typeset_order(cp,tp->comline);
@@ -1333,7 +1351,7 @@ static Shnode_t	*item(Lex_t *lexp,int flag)
 	    case 0:
 		t = (Shnode_t*)simple(lexp,flag,io);
 		if(t->com.comarg && lexp->intypeset)
-			check_typedef(&t->com);
+			check_typedef(&t->com, lexp->intypeset);
 		lexp->intypeset = 0;
 		lexp->inexec = 0;
 		t->tre.tretyp |= showme;
@@ -1454,6 +1472,8 @@ static Shnode_t *simple(Lex_t *lexp,int flag, struct ionod *io)
 						assignment = 1;
 						if(np >= SYSTYPESET && np <= SYSTYPESET_END)
 							lexp->intypeset = 1;
+						else if(np == SYSENUM)
+							lexp->intypeset = 2;
 						key_on = 1;
 					}
 					else if(np==SYSCOMMAND)	/* treat 'command typeset', etc. as declaration command */

src/cmd/ksh93/tests/enum.sh

@@ -80,8 +80,7 @@ got=$(set +x; redirect 2>&1; Color_t -A clr=([foo]=red [bar]=blue [bad]=BAD); pr
 	"expected status 1, *$(printf %q "$exp"); got status $e, $(printf %q "$got")"
 
 # associative enum array
-# (need 'eval' to delay parsing when testing with shcomp, as it parses the entire script without executing the type definition)
-eval 'Color_t -A Colors=([foo]=red [bar]=blue [bad]=green [zut]=orange [blauw]=blue [rood]=red [groen]=green [geel]=yellow)'
+Color_t -A Colors=([foo]=red [bar]=blue [bad]=green [zut]=orange [blauw]=blue [rood]=red [groen]=green [geel]=yellow)
 exp='green blue blue red yellow green red orange'
 got=${Colors[@]}
 [[ $got == "$exp" ]] || err_exit "\${array[@]} doesn't yield all values for associative enum arrays" \
@@ -95,8 +94,7 @@ got=$(typeset -p Colors)
 [[ -n $got ]] && err_exit "unsetting associative enum array does not work (got $(printf %q "$got"))"
 
 # indexed enum array
-# (need 'eval' to delay parsing when testing with shcomp, as it parses the entire script without executing the type definition)
-eval 'Color_t -a iColors=(red blue green orange blue red green yellow)'
+Color_t -a iColors=(red blue green orange blue red green yellow)
 exp='red blue green orange blue red green yellow'
 got=${iColors[@]}
 [[ $got == "$exp" ]] || err_exit "\${array[@]} doesn't yield all values for indexed enum arrays" \
@@ -117,5 +115,23 @@ got="${testarray[3]:-BUG} ${testarray[@]:-BUG}"
 [[ $got == "$exp" ]] || err_exit "assigning first enum element to indexed array failed" \
 	"(expected $(printf %q "$exp"); got $(printf %q "$got"))"
 
+# ======
+# https://github.com/ksh93/ksh/issues/256
+cmd='enum Color_t=(red green blue); Color_t -A Colors=([rubie]=red [verde]=green [blau]=blue); typeset -p Colors'
+exp='Color_t -A Colors=([blau]=blue [rubie]=red [verde]=green)'
+got=$("$SHELL" -c "$cmd" 2>&1)
+[[ $got == "$exp" ]] || err_exit "-c failed" \
+	"(expected $(printf %q "$exp"); got $(printf %q "$got"))"
+got=$("$SHELL" -c "eval '$cmd'" 2>&1)
+[[ $got == "$exp" ]] || err_exit "-c script with eval failed" \
+	"(expected $(printf %q "$exp"); got $(printf %q "$got"))"
+print "$cmd" >cmd.sh
+got=$("$SHELL" -c '. ./cmd.sh' 2>&1)
+[[ $got == "$exp" ]] || err_exit "dotted script failed" \
+	"(expected $(printf %q "$exp"); got $(printf %q "$got"))"
+got=$("$SHELL" -c 'source ./cmd.sh' 2>&1)
+[[ $got == "$exp" ]] || err_exit "sourced script failed" \
+	"(expected $(printf %q "$exp"); got $(printf %q "$got"))"
+
 # ======
 exit $((Errors<125?Errors:125))