Fix bugs related to --posix shell option (re: 921bbcae, f45a0f16)

This fixes the following:
1. 'set --posix' now works as an equivalent of 'set -o posix'.
2. The posix option turns off braceexpand and turns on letoctal.
   Any attempt to override that in a single command such as 'set -o
   posix +o letoctal' was quietly ignored. This now works as long
   as the overriding option follows the posix option in the command.
3. The --default option to 'set' now stops the 'posix' option, if
   set or unset in the same 'set' command, from changing other
   options. This allows the command output by 'set +o' to correctly
   restore the current options.

src/cmd/ksh93/data/builtins.c:
- To make 'set --posix' work, we must explicitly list it in
  sh_set[] as a supported option so that AST optget(3) recognises
  it and won't override it with its own default --posix option,
  which converts the optget(3) string to at POSIX getopt(3) string.
    This means it will appear as a separate entry in --man output,
  whether we want it to or not. So we might as well use it as an
  example to document how --optionname == -o optionname, replacing
  the original documentation that was part of the '-o' description.

src/cmd/ksh93/sh/args.c: sh_argopts():
- Add handling for explitit --posix option in data/builtins.c.
- Move SH_POSIX syncing SH_BRACEEXPAND and SH_LETOCTAL from
  sh_applyopts() into the option parsing loop here. This fixes
  the bug that letoctal was ignored in 'set -o posix +o letoctal'.
- Remember if --default was used in a flag, and do not sync options
  with SH_POSIX if the flag is set. This makes 'set +o' work.

src/cmd/ksh93/include/argnod.h,
src/cmd/ksh93/data/msg.c,
src/cmd/ksh93/sh/args.c: sh_printopts():
- Do not potentially translate the 'on' and 'off' labels in 'set
  -o' output. No other shell does, and some scripts parse these.

src/cmd/ksh93/sh/init.c: sh_init():
- Turn on SH_LETOCTAL early along with SH_POSIX if the shell was
  invoked as sh; this makes 'sh -o' and 'sh +o' show expected
  options (not that anyone does this, but correctness is good).

src/cmd/ksh93/include/defs.h,
src/cmd/ksh93/include/shell.h:
- The state flags were in defs.h and most (but not all) of the
  shell options were in shell.h. Gather all the shell state and
  option flag definitions into one place in shell.h for clarity.
- Remove unused SH_NOPROFILE and SH_XARGS option flags.

src/cmd/ksh93/tests/options.sh:
- Add tests for these bugs.

src/lib/libast/misc/optget.c: styles[]:
- Edit default optget(3) option self-documentation for clarity.

Several changed files:
- Some SHOPT_PFSH fixes to avoid compiling dead code.

NEWS

@@ -10,6 +10,17 @@ Any uppercase BUG_* names are modernish shell bug IDs.
   an additional check against the system clock to make sure it sleeps at least
   the given amount of time. Thanks to Lev Kujawski for adding this feature.
 
+- A few bugs were fixed that 93u+m introduced along with the new '-o posix'
+  shell option on 2020-09-01:
+  1. 'set --posix' now works as the expected equivalent of 'set -o posix'.
+  2. As of 2020-09-18, the posix option turns off braceexpand and turns on
+     letoctal. Any attempt to override that in a single command such as
+     'set -o posix +o letoctal' was quietly ignored. This now works as long
+     as the overriding option follows the posix option on the command line.
+  3. The --default option to 'set' now stops the 'posix' option, if set or
+     unset in the same 'set' command, from changing other options. This allows
+     the command output by 'set +o' to correctly restore the current options.
+
 2021-02-11:
 
 - Fixed a bug that caused ksh to lose track of all running background jobs if

src/cmd/ksh93/bltins/typeset.c

@@ -1055,8 +1055,10 @@ int	b_builtin(int argc,char *argv[],Shbltin_t *context)
 	{
 		if(sh_isoption(SH_RESTRICTED))
 			errormsg(SH_DICT,ERROR_exit(1),e_restricted,argv[-opt_info.index]);
+#if SHOPT_PFSH
 		if(sh_isoption(SH_PFSH))
 			errormsg(SH_DICT,ERROR_exit(1),e_pfsh,argv[-opt_info.index]);
+#endif
 		if(tdata.sh->subshell && !tdata.sh->subshare)
 			sh_subfork();
 	}

src/cmd/ksh93/data/builtins.c

@@ -196,14 +196,12 @@ const char sh_set[] =
 	"control.]"
 "[n?The shell reads commands and checks for syntax errors, but does "
 	"not execute the command.  Usually specified on command invocation.]"
-"[o]:?[option?If \aoption\a is not specified, the list of options and "
-	"their current settings will be written to standard output.  When "
-	"invoked with a \b+\b the options will be written in a format "
-	"that can be reinput to the shell to restore the settings. "
-	"Options \b-o\b \aname\a can also be specified with \b--\b\aname\a "
-	"and \b+o \aname\a can be specified with \b--no\b\aname\a  except that "
-	"options names beginning with \bno\b are turned on by omitting \bno\b."
-	"This option can be repeated to enable/disable multiple options. "
+"[o]:?[option?A \b-o\b with no \aoption\a will write the list of options and "
+	"their current settings to standard output. "
+	"A \b+o\b with no \aoption\a writes a command that the shell can run "
+	"to restore the current options state. "
+	"\b-o\b \aoption\a turns on \aoption\a and \b+o\b \aoption\a turns it "
+	"off. This can be repeated to enable/disable multiple options. "
 	"The value of \aoption\a must be one of the following:]{"
 		"[+allexport?Equivalent to \b-a\b.]"
 		"[+bgnice?Runs background jobs at lower priorities.]"
@@ -258,6 +256,15 @@ const char sh_set[] =
 #endif
 		"[+xtrace?Equivalent to \b-x\b.]"
 "}"
+/*
+ * --posix is an AST optget(3) default option, so for ksh to use it, it must be listed
+ * explicitly (and handled by sh_argopts() in sh/args.c) to stop optget(3) overriding it.
+ * Since it must appear here, use it as an example to document how --option == -o option.
+ */
+"[05:posix?For any \b-o\b option (such as \bposix\b), \b--posix\b is equivalent "
+	"to \b-o posix\b and \b--noposix\b is equivalent to \b+o posix\b. "
+	"However, option names with a \bno\b prefix "
+	"are turned on by omitting \bno\b.]"
 "[p?Privileged mode.  Disabling \b-p\b sets the effective user id to the "
 	"real user id, and the effective group id to the real group id.  "
 	"Enabling \b-p\b restores the effective user and group ids to their "

src/cmd/ksh93/data/msg.c

@@ -67,7 +67,9 @@ const char e_subst[]		= "%s: bad substitution";
 const char e_create[]		= "%s: cannot create";
 const char e_tmpcreate[]	= "cannot create temporary file";
 const char e_restricted[]	= "%s: restricted";
+#if SHOPT_PFSH
 const char e_pfsh[]		= "%s: disabled in profile shell";
+#endif
 const char e_pexists[]		= "process already exists";
 const char e_exists[]		= "%s: file already exists";
 const char e_pipe[]		= "cannot create pipe";
@@ -133,8 +135,6 @@ const char e_overlimit[]	= "%s: limit exceeded";
 const char e_badsyntax[]	= "incorrect syntax";
 const char e_badwrite[]		= "write to %d failed";
 const char e_staticfun[]	= "%s: defined as a static function in type %s and cannot be redefined";
-const char e_on	[]		= "on";
-const char e_off[]		= "off";
 const char is_reserved[]	= " is a keyword";
 const char is_builtin[]		= " is a shell builtin";
 const char is_spcbuiltin[]	= " is a special shell builtin";

src/cmd/ksh93/include/argnod.h

@@ -133,8 +133,6 @@ extern int		sh_argopts(int,char*[],void*);
 
 
 extern const char	e_heading[];
-extern const char	e_off[];
-extern const char	e_on[];
 extern const char	e_sptbnl[];
 extern const char	e_subst[];
 extern const char	e_option[];

src/cmd/ksh93/include/defs.h

@@ -297,36 +297,6 @@ struct shared
 #define SH_CMDLIB_DIR	"/opt/ast/bin"
 #endif
 
-/* states */
-/* low numbered states are same as options */
-#define SH_NOFORK	0	/* set when fork not necessary */
-#define	SH_FORKED	7	/* set when process has been forked */
-#define	SH_PROFILE	8	/* set when processing profiles */
-#define SH_NOALIAS	9	/* do not expand non-exported aliases */
-#define SH_NOTRACK	10	/* set to disable sftrack() function */
-#define SH_STOPOK	11	/* set for stopable builtins */
-#define SH_GRACE	12	/* set for timeout grace period */
-#define SH_TIMING	13	/* set while timing pipelines */
-#define SH_DEFPATH	14	/* set when using default path */
-#define SH_INIT		15	/* set when initializing the shell */
-#define SH_TTYWAIT	16	/* waiting for keyboard input */ 
-#define	SH_FCOMPLETE	17	/* set for filename completion */
-#define	SH_PREINIT	18	/* set with SH_INIT before parsing options */
-#define SH_COMPLETE	19	/* set for command completion */
-#define SH_INTESTCMD	20	/* set while test/[ command is being run */
-#define SH_XARG		21	/* set while in xarg (command -x) mode */
-
-#if SHOPT_BRACEPAT
-#define SH_BRACEEXPAND		42
-#endif
-#define SH_POSIX		46
-#define SH_MULTILINE    	47
-
-#define SH_NOPROFILE		78
-#define SH_NOUSRPROFILE		79
-#define SH_LOGIN_SHELL		67
-#define SH_COMMANDLINE		0x100	/* flag for invocation-only options ('set -o' cannot change them) */
-
 #define SH_ID			"ksh"	/* ksh id */
 #define SH_STD			"sh"	/* standard sh id */
 

src/cmd/ksh93/include/path.h

@@ -109,7 +109,9 @@ extern const char e_timeformat[];
 extern const char e_badtformat[];
 extern const char e_dot[];
 extern const char e_funload[];
+#if SHOPT_PFSH
 extern const char e_pfsh[];
+#endif
 extern const char e_pwd[];
 extern const char e_logout[];
 extern const char e_alphanum[];

src/cmd/ksh93/include/shell.h

@@ -60,6 +60,33 @@ typedef void	(*Shinit_f)(Shell_t*, int);
 union Shnode_u;
 typedef union Shnode_u Shnode_t;
 
+/*
+ * Shell state flags. Used with sh_isstate(), sh_onstate(), sh_offstate().
+ * See also shell options below. States 0-5 are also used as shell options.
+ */
+#define SH_NOFORK	0	/* set when fork not necessary */
+#define	SH_FORKED	7	/* set when process has been forked */
+#define	SH_PROFILE	8	/* set when processing profiles */
+#define SH_NOALIAS	9	/* do not expand non-exported aliases */
+#define SH_NOTRACK	10	/* set to disable sftrack() function */
+#define SH_STOPOK	11	/* set for stopable builtins */
+#define SH_GRACE	12	/* set for timeout grace period */
+#define SH_TIMING	13	/* set while timing pipelines */
+#define SH_DEFPATH	14	/* set when using default path */
+#define SH_INIT		15	/* set when initializing the shell */
+#define SH_TTYWAIT	16	/* waiting for keyboard input */
+#define	SH_FCOMPLETE	17	/* set for filename completion */
+#define	SH_PREINIT	18	/* set with SH_INIT before parsing options */
+#define SH_COMPLETE	19	/* set for command completion */
+#define SH_INTESTCMD	20	/* set while test/[ command is being run */
+#define SH_XARG		21	/* set while in xarg (command -x) mode */
+
+/*
+ * Shell options (set -o). Used with sh_isoption(), sh_onoption(), sh_offoption().
+ * There can be a maximum of 256 (0..0xFF) shell options.
+ * The short option letters are defined in optksh[] and flagval[] in sh/args.c.
+ * The long option names are defined in shtab_options[] in data/options.c.
+ */
 #define SH_CFLAG	0
 #define SH_HISTORY	1	/* used also as a state */
 #define	SH_ERREXIT	2	/* used also as a state */
@@ -72,7 +99,9 @@ typedef union Shnode_u Shnode_t;
 #define SH_NOUNSET	9
 #define SH_NOGLOB	10
 #define SH_ALLEXPORT	11
+#if SHOPT_PFSH
 #define SH_PFSH		12
+#endif
 #define SH_IGNOREEOF	13
 #define SH_NOCLOBBER	14
 #define SH_MARKDIRS	15
@@ -95,10 +124,17 @@ typedef union Shnode_u Shnode_t;
 #define SH_DICTIONARY	30
 #define SH_PIPEFAIL	32
 #define SH_GLOBSTARS	33
-#define SH_XARGS	34
 #define SH_RC		35
 #define SH_SHOWME	36
 #define SH_LETOCTAL	37
+#if SHOPT_BRACEPAT
+#define SH_BRACEEXPAND	42
+#endif
+#define SH_POSIX	46
+#define SH_MULTILINE	47
+#define SH_LOGIN_SHELL	67
+#define SH_NOUSRPROFILE	79	/* internal use only */
+#define SH_COMMANDLINE	0x100	/* bit flag for invocation-only options ('set -o' cannot change them) */
 
 /*
  * passed as flags to builtins in Nambltin_t struct when BLT_OPTIM is on

src/cmd/ksh93/include/shtable.h

@@ -53,6 +53,7 @@ struct shtable3
 #define sh_lookup(name,value)	(sh_locate(name,(Shtable_t*)(value),sizeof(*(value)))->sh_number)
 extern const Shtable_t		shtab_testops[];
 extern const Shtable_t		shtab_options[];
+extern const Shtable_t		shtab_options_posix[];
 extern const Shtable_t		shtab_attributes[];
 extern const struct shtable2	shtab_variables[];
 extern const struct shtable2	shtab_aliases[];

src/cmd/ksh93/include/version.h

@@ -20,7 +20,7 @@
 
 #define SH_RELEASE_FORK	"93u+m"		/* only change if you develop a new ksh93 fork */
 #define SH_RELEASE_SVER	"1.0.0-alpha"	/* semantic version number: https://semver.org */
-#define SH_RELEASE_DATE	"2021-02-11"	/* must be in this format for $((.sh.version)) */
+#define SH_RELEASE_DATE	"2021-02-14"	/* must be in this format for $((.sh.version)) */
 #define SH_RELEASE_CPYR	"(c) 2020-2021 Contributors to ksh " SH_RELEASE_FORK
 
 /* Scripts sometimes field-split ${.sh.version}, so don't change amount of whitespace. */

src/cmd/ksh93/sh/args.c

@@ -128,7 +128,7 @@ int sh_argopts(int argc,register char *argv[], void *context)
 	Lex_t		*lp = (Lex_t*)(shp->lex_context);
 #endif
 	Shopt_t		newflags;
-	int setflag=0, action=0, trace=(int)sh_isoption(SH_XTRACE);
+	int		defaultflag=0, setflag=0, action=0, trace=(int)sh_isoption(SH_XTRACE);
 	Namval_t *np = NIL(Namval_t*);
 	const char *cp;
 	int verbose,f;
@@ -173,6 +173,12 @@ int sh_argopts(int argc,register char *argv[], void *context)
 			if(sh_isoption(SH_RESTRICTED) && !f && o==SH_RESTRICTED)
 				errormsg(SH_DICT,ERROR_exit(1), e_restricted, opt_info.arg);
 			break;
+		    case -5:	/* --posix must be handled explicitly to stop AST optget(3) overriding it */
+			if(opt_info.num)
+				on_option(&newflags,SH_POSIX);
+			else
+				off_option(&newflags,SH_POSIX);
+			break;
 		    case -6:	/* --default */
 			{
 				register const Shtable_t *tp;
@@ -180,8 +186,9 @@ int sh_argopts(int argc,register char *argv[], void *context)
 					if(!(o&SH_COMMANDLINE) && is_option(&newflags,o&0xff))
 						off_option(&newflags,o&0xff);
 			}
+			defaultflag++;
 		    	continue;
-	 	    case -7:
+		    case -7:	/* --state */
 			f = 0;
 		    	goto byname;
 	 	    case 'D':
@@ -249,8 +256,14 @@ int sh_argopts(int argc,register char *argv[], void *context)
 				off_option(&newflags,SH_EMACS);
 				off_option(&newflags,SH_GMACS);
 			}
-
 #endif /* SHOPT_ESH && SHOPT_VSH */
+			if(o==SH_POSIX && !defaultflag)
+			{
+#if SHOPT_BRACEPAT
+				off_option(&newflags,SH_BRACEEXPAND);
+#endif
+				on_option(&newflags,SH_LETOCTAL);
+			}
 			on_option(&newflags,o);
 			off_option(&ap->sh->offoptions,o);
 		}
@@ -258,6 +271,13 @@ int sh_argopts(int argc,register char *argv[], void *context)
 		{
 			if ((o == SH_RESTRICTED) && sh_isoption(SH_RESTRICTED))
 				errormsg(SH_DICT,ERROR_exit(1),e_restricted,"r"); /* set -r cannot be unset */
+			if(o==SH_POSIX && !defaultflag)
+			{
+#if SHOPT_BRACEPAT
+				on_option(&newflags,SH_BRACEEXPAND);
+#endif
+				off_option(&newflags,SH_LETOCTAL);
+			}
 			if(o==SH_XTRACE)
 				trace = 0;
 			off_option(&newflags,o);
@@ -365,21 +385,6 @@ void sh_applyopts(Shell_t* shp,Shopt_t newflags)
 		sh_onstate(SH_MONITOR);
 	else if(sh_isoption(SH_MONITOR) && !is_option(&newflags,SH_MONITOR))
 		sh_offstate(SH_MONITOR);
-	/* -o posix also switches -o braceexpand and -o letoctal */
-	if(!sh_isoption(SH_POSIX) && is_option(&newflags,SH_POSIX))
-	{
-#if SHOPT_BRACEPAT
-		off_option(&newflags,SH_BRACEEXPAND);
-#endif
-		on_option(&newflags,SH_LETOCTAL);
-	}
-	else if(sh_isoption(SH_POSIX) && !is_option(&newflags,SH_POSIX))
-	{
-#if SHOPT_BRACEPAT
-		on_option(&newflags,SH_BRACEEXPAND);
-#endif
-		off_option(&newflags,SH_LETOCTAL);
-	}
 	shp->options = newflags;
 }
 
@@ -606,7 +611,7 @@ void sh_printopts(Shopt_t oflags,register int mode, Shopt_t *mask)
 		{
 			sfputr(sfstdout,name,' ');
 			sfnputc(sfstdout,' ',24-strlen(name));
-			sfputr(sfstdout,on ? sh_translate(e_on) : sh_translate(e_off),'\n');
+			sfputr(sfstdout,on ? "on" : "off",'\n');
 		}
 		else if(mode&PRINT_ALL) /* print unset options also */
 			sfprintf(sfstdout, "set %co %s\n", on?'-':'+', name);

src/cmd/ksh93/sh/init.c

@@ -1307,7 +1307,10 @@ Shell_t *sh_init(register int argc,register char *argv[], Shinit_f userinit)
 		if(type&SH_TYPE_LOGIN)
 			shp->login_sh = 2;
 		if(type&SH_TYPE_POSIX)
+		{
 			sh_onoption(SH_POSIX);
+			sh_onoption(SH_LETOCTAL);
+		}
 	}
 	/* read the environment; don't import attributes yet, but save pointer to them */
 	save_envmarker = env_init(shp);

src/cmd/ksh93/sh/path.c

@@ -212,7 +212,11 @@ static pid_t path_xargs(Shell_t *shp,const char *path, char *argv[],char *const
 				saveargs = 0;
 			}
 		}
+#if SHOPT_PFSH
 		else if(spawn && !sh_isoption(SH_PFSH))
+#else
+		else if(spawn)
+#endif
 		{
 			shp->xargexit = exitval;
 			if(saveargs)
@@ -1179,7 +1183,11 @@ pid_t path_spawn(Shell_t *shp,const char *opath,register char **argv, char **env
 		path = sp;
 	}
 #endif /* SHELLMAGIC */
+#if SHOPT_PFSH
 	if(spawn && !sh_isoption(SH_PFSH))
+#else
+	if(spawn)
+#endif
 		pid = _spawnveg(shp,opath, &argv[0],envp, spawn>>1);
 	else
 		pid = path_pfexecve(shp,opath, &argv[0] ,envp,spawn);

src/cmd/ksh93/tests/options.sh

@@ -534,16 +534,13 @@ print $'alias print=:\nprint foobar' > dotfile
 [[ $(ENV=/.$PWD/envfile $SHELL -i -c : 2>/dev/null) == foobar ]] && err_exit 'files source from profile does not process aliases correctly'
 
 # ======
-# test that '-o posix' option (not having a letter) does not affect "$-" expansion
-# other than B = braceexpand
 if [[ -o ?posix ]]; then
-(
-	command set +o posix 2>/dev/null
-	opt1=${-/B/}
-	command set -o posix 2>/dev/null
-	opt2=$-
-	[[ $opt1 == "$opt2" ]]
-) || err_exit '-o posix option affects $- expansion'
+	(set +o posix; o1=${-/B/}; set -o posix; o2=${-/B/}; [[ $o1 == "$o2" ]]) || err_exit 'set -o posix affects $- expansion'
+	(set +o posix; set --posix >/dev/null; [[ -o posix ]]) || err_exit "set --posix != set -o posix"
+	(set -o posix; set --noposix; [[ -o posix ]]) && err_exit "set --noposix != set +o posix"
+	(set -o posix +o letoctal; [[ -o letoctal ]]) && err_exit "failed to stop posix option from turning on letoctal"
+	(set +B; set -o posix -B; [[ -o braceexpand ]]) || err_exit "failed to stop posix option from turning off bracceexpand"
+	(set --default -o posix; [[ -o letoctal ]]) && err_exit "set --default failed to stop posix option from changing others"
 fi
 
 # ======

src/lib/libast/misc/optget.c

@@ -229,29 +229,29 @@ standard error."),
 static const Help_t	styles[] =
 {
 	C("about"),	"-",		STYLE_match,
-	Z("List all implementation info."),
+	Z("Show all implementation info."),
 	C("api"),	"?api",		STYLE_api,
-	Z("List detailed info in program readable form."),
+	Z("Output detailed info in program readable form."),
 	C("help"),	"",		-1,
-	Z("List detailed help option info."),
+	Z("Show detailed help option info."),
 	C("html"),	"?html",	STYLE_html,
-	Z("List detailed info in html."),
+	Z("Output detailed info in HTML."),
 	C("keys"),	"?keys",	STYLE_keys,
-	Z("List the usage translation key strings with C style escapes."),
+	Z("Output usage key strings for translation."),
 	C("long"),	"?long",	STYLE_long,
-	Z("List long option usage."),
+	Z("Show brief usage with long options."),
 	C("man"),	"?man",		STYLE_man,
-	Z("List detailed info in displayed man page form."),
+	Z("Show detailed info as a manual page."),
 	C("nroff"),	"?nroff",	STYLE_nroff,
-	Z("List detailed info in nroff."),
+	Z("Output detailed info in nroff format."),
 	C("options"),	"?options",	STYLE_options,
 	Z("List short and long option details."),
 	C("posix"),	"?posix",	STYLE_posix,
-	Z("List posix getopt usage."),
+	Z("Output POSIX-compliant getopt(3) short options string."),
 	C("short"),	"?short",	STYLE_short,
-	Z("List short option usage."),
+	Z("Show brief usage with short options."),
 	C("usage"),	"?usage",	STYLE_usage,
-	Z("List the usage string with C style escapes."),
+	Z("Output the full AST optget(3) usage string."),
 };
 
 static const List_t	help_tail[] =