test/[/[[: Fix incorrect 0-skipping, float precision (re: c734568b)

The arguments to the binary numerical comparison operators (-eq,
-gt, etc.) in the [[ and test/[ commands are treated as arithmetic
expressions, even if $((...)) is not used. But there is some
seriously incorrect behaviour:

Reproducers (all should output 0/true):

    $ [[ 0x0A -eq 10 ]]; echo $?
    1
    $ [[ 1+0x0A -eq 11 ]]; echo $?
    0
    $ (set --posix; [[ 010 -eq 8 ]]); echo $?
    1
    $ (set --posix; [[ +010 -eq 8 ]]); echo $?
    0
    $ [[ 0xA -eq 10 ]]; echo $?
    1
    $ xA=10; [[ 0xA -eq 10 ]]; echo $?
    0
    $ xA=WTF; [[ 0xA -eq 10 ]]; echo $?
    ksh: WTF: parameter not set

(POSIX mode enables the recognition of the initial 0 as a prefix
indicating an octal number in arithmetic expressions.)

The cause is the two 'while' loops in this section in test_binop()
in src/cmd/ksh93/bltins/test.c:

502:	if(op&TEST_ARITH)
503:	{
504:		while(*left=='0')
505:			left++;
506:		while(*right=='0')
507:			right++;
508:		lnum = sh_arith(left);
509:		rnum = sh_arith(right);
510:	}

So initial zeros are unconditionally skipped. Ostensibly this is to
disable the recognition of the initial zero as an octal prefix as
well as 0x as a hexadecimal prefix. This would be okay for
enforcing a decimal-only limitation for simple numbers, but to do
this for arithmetic expressions is flagrantly incorrect, to say the
least. (insert standard rant about AT&T quality standards here)

The fix for '[[' is simply to delete the two 'while' loops. But
that creates a problem for the deprecated-but-standard 'test'/'['
command, which also uses the test_binop() function. According to
POSIX, test/[ only parses simple decimal numbers, so octal, etc. is
not a thing. But, after that fix, 'test 08 -eq 10' in POSIX mode
yields true, which is unlike every other shell. (Note that this is
unlike [[ 08 -eq 10 ]], which yields true on bash because '[['
parses operands as arithmetic expressions.)

For test/[ in non-POSIX mode, we don't need to change anything. For
POSIX mode, we should only parse literal decimal numbers for these
operators in test/[, disallowing unexpanded arithmetic expressions.
This makes ksh's POSIX-mode test/[ act like every other shell and
like external .../bin/test commands shipped by OSs.

src/cmd/ksh93/bltins/text.c: test_binop():
- Correct a type mismatch. The left and right hand numbers should
  be Sfdouble_t, the maximum double length on the current system,
  and the type that sh_arith() returns. Instead, long double
  (typeset -lF) values were reduced to double (typeset -F) before
  comparing!
- For test/[ in POSIX, only accept simple decimal numbers: use
  strtold() instead of sh_arith(). Do skip initial zeros here as
  this disables the recognition of the hexadecimal prefix. Throw an
  error on an invalid decimal number. Floating point constants are
  still parsed, but that's fine as this does not cause any
  incompatibility with POSIX.

- For code legibility, move handling of TEST_EQ, etc. to within the
  if(op&TEST_ARITH) block. This also allows lnum and rnum to be
  local to that block.

NEWS

@@ -3,6 +3,15 @@ For full details, see the git log at: https://github.com/ksh93/ksh/tree/1.0
 
 Any uppercase BUG_* names are modernish shell bug IDs.
 
+2022-07-26:
+
+- Fixed incorrect handling of initial zeros in test/[ and [[ arithmetic
+  comparison operators. For example, [[ 0x0A -eq 10 ]] yielded false but
+  [[ 1+0x0A -eq 11 ]] yielded true.
+
+- Fixed comparing long floating point (typeset -lF) values in test/[ and [[;
+  the values were reduced to standard floating point before comparing.
+
 2022-07-25:
 
 - Release candidate: ksh 93u+m/1.0.0-rc.1.

src/cmd/ksh93/bltins/test.c

@@ -498,15 +498,48 @@ int test_unop(register int op,register const char *arg)
  */
 int test_binop(register int op,const char *left,const char *right)
 {
-	register double lnum = 0, rnum = 0;
 	if(op&TEST_ARITH)
 	{
-		while(*left=='0')
-			left++;
-		while(*right=='0')
-			right++;
-		lnum = sh_arith(left);
-		rnum = sh_arith(right);
+		Sfdouble_t lnum, rnum;
+		if(sh.bltinfun==b_test && sh_isoption(SH_POSIX))
+		{
+			/* for test/[ in POSIX, only accept simple decimal numbers */
+			char *l = (char*)left, *r = (char*)right;
+			while(*l=='0')
+				l++;
+			while(*r=='0')
+				r++;
+			lnum = strtold(l,&l);
+			rnum = strtold(r,&r);
+			if(*l || *r)
+			{
+				errormsg(SH_DICT, ERROR_exit(2), e_number, *l ? left : right);
+				UNREACHABLE();
+			}
+		}
+		else
+		{
+			/* numeric operands are arithmetic expressions */
+			lnum = sh_arith(left);
+			rnum = sh_arith(right);
+		}
+		switch(op)
+		{
+			case TEST_EQ:
+				return(lnum==rnum);
+			case TEST_NE:
+				return(lnum!=rnum);
+			case TEST_GT:
+				return(lnum>rnum);
+			case TEST_LT:
+				return(lnum<rnum);
+			case TEST_GE:
+				return(lnum>=rnum);
+			case TEST_LE:
+				return(lnum<=rnum);
+		}
+		/* all arithmetic binary operators should be covered above */
+		UNREACHABLE();
 	}
 	switch(op)
 	{
@@ -534,20 +567,8 @@ int test_binop(register int op,const char *left,const char *right)
 			return(test_time(left,right)>0);
 		case TEST_OT:
 			return(test_time(left,right)<0);
-		case TEST_EQ:
-			return(lnum==rnum);
-		case TEST_NE:
-			return(lnum!=rnum);
-		case TEST_GT:
-			return(lnum>rnum);
-		case TEST_LT:
-			return(lnum<rnum);
-		case TEST_GE:
-			return(lnum>=rnum);
-		case TEST_LE:
-			return(lnum<=rnum);
 	}
-	/* all possible binary operators should be covered above */
+	/* all non-arithmetic binary operators should be covered above */
 	UNREACHABLE();
 }
 

src/cmd/ksh93/data/testops.c

@@ -56,7 +56,7 @@ const Shtable_t shtab_testops[] =
 };
 
 const char sh_opttest[] =
-"[-1c?\n@(#)$Id: test (ksh 93u+m) 2022-03-10 $\n]"
+"[-1c?\n@(#)$Id: test (ksh 93u+m) 2022-07-25 $\n]"
 "[--catalog?" SH_DICT "]"
 "[+NAME?test, [ - evaluate expression]"
 "[+DESCRIPTION?\btest\b evaluates expressions and returns its result using the "

src/cmd/ksh93/include/version.h

@@ -22,8 +22,8 @@
 #include <releaseflags.h>
 
 #define SH_RELEASE_FORK	"93u+m"		/* only change if you develop a new ksh93 fork */
-#define SH_RELEASE_SVER	"1.0.0-rc.1"	/* semantic version number: https://semver.org */
-#define SH_RELEASE_DATE	"2022-07-25"	/* must be in this format for $((.sh.version)) */
+#define SH_RELEASE_SVER	"1.0.0-rc.2"	/* semantic version number: https://semver.org */
+#define SH_RELEASE_DATE	"2022-07-26"	/* must be in this format for $((.sh.version)) */
 #define SH_RELEASE_CPYR	"(c) 2020-2022 Contributors to ksh " SH_RELEASE_FORK
 
 /* Scripts sometimes field-split ${.sh.version}, so don't change amount of whitespace. */

src/cmd/ksh93/sh.1

@@ -7790,6 +7790,18 @@ built-ins;
 stops the \fB.\fR command (but not \fBsource\fR) from looking up functions
 defined with the \fBfunction\fR syntax;
 .IP \[bu]
+disables the recognition of unexpanded shell arithmetic expressions in the
+numerical comparison operators
+.BR \-eq ,
+.BR \-ne ,
+.BR \-gt ,
+.BR \-ge ,
+.B \-lt
+and
+.B \-le
+of the \fBtest\fR/\fB[\fR built-in command, causing them to accept only
+decimal numbers as operands;
+.IP \[bu]
 changes the \fBtest\fR/\fB[\fR built-in command to make its deprecated
 \fIexpr1\fR \fB-a\fR \fIexpr2\fR and \fIexpr1\fR \fB-o\fR \fIexpr2\fR operators
 work even if \fIexpr1\fR equals "\fB!\fR" or "\fb(\fR" (which means the
@@ -8001,7 +8013,7 @@ to those specified for the \f3[[\fP compound command under
 .I Conditional Expressions
 above, but with several important differences. The \f3=\fP, \f3==\fP and
 \f3!=\fP operators test for string (in)equality without pattern matching;
-\f3==\fP is nonstandard and unportable. The f3&&\fP and \f3||\fP operators are
+\f3==\fP is nonstandard and unportable. The \f3&&\fP and \f3||\fP operators are
 not available. Instead, the \f3-a\fP and \f3-o\fP binary operators can be
 used, but they are fraught with pitfalls due to grammatical ambiguities and
 therefore deprecated in favor of invoking separate \f3test\fP commands. Most

src/cmd/ksh93/tests/bracket.sh

@@ -503,5 +503,20 @@ command eval 'x=([x]=1 [y)' 2>/dev/null
 [[ -z $x ]] 2>/dev/null || err_exit "[[ ... ]] breaks after syntax error in associative array assignment (got status $?)"
 PATH=$savePATH
 
+# ======
+# Two more shining examples of superior AT&T quality standards :P
+
+x0A=WTF
+unset WTF
+got=$([[ 0x0A -eq 010 ]] 2>&1) || err_exit "0x0A != 010 in [[ (got $(printf %q "$got"))"
+got=$(test 0x0A -eq 010 2>&1) || err_exit "0x0A != 010 in test (got $(printf %q "$got"))"
+unset XA
+
+typeset -lF x=18446744073709551615 y=x+1
+if	((x != y))
+then	[[ x -eq y ]] && err_exit "comparing long floats fails"
+fi
+unset x y
+
 # ======
 exit $((Errors<125?Errors:125))

src/cmd/ksh93/tests/posix.sh

@@ -196,9 +196,16 @@ let "017 == 15" || err_exit "leading octal zero not recognised in 'let' in --pos
 (set --noposix; let "017 == 17") || err_exit "leading octal zero erroneously recognised in --noposix"
 (set --noposix --letoctal; let "017 == 15") || err_exit "leading octal zero not recognised in --noposix --letoctal (1)"
 (set --noposix; set --letoctal; let "017 == 15") || err_exit "leading octal zero not recognised in --noposix --letoctal (2)"
+test 010 -eq 10 || err_exit "'test' not ignoring leading octal zero in --posix"
+[ 010 -eq 10 ] || err_exit "'[' not ignoring leading octal zero in --posix"
+[[ 010 -eq 8 ]] || err_exit "'[[' ignoring leading octal zero in --posix"
+(set --noposix; [[ 010 -eq 10 ]]) || err_exit "'[[' not ignoring leading octal zero in --noposix"
 
 # disables zero-padding of seconds in the output of the time and times built-ins;
-exp=$'^user\t0m0.[0-9]{2}s\nsys\t0m0.[0-9]{2}s\n0m0.[0-9]{3}s 0m0.[0-9]{3}s\n0m0.000s 0m0.000s$'
+case ${.sh.version} in
+*93u+m/1.0.*)	exp=$'^user\t0m0.[0-9]{2}s\nsys\t0m0.[0-9]{2}s\n0m0.[0-9]{3}s 0m0.[0-9]{3}s\n0m0.000s 0m0.000s$' ;;
+*)		exp=$'^user\t0m0.[0-9]{3}s\nsys\t0m0.[0-9]{3}s\n0m0.[0-9]{3}s 0m0.[0-9]{3}s\n0m0.000s 0m0.000s$' ;;
+esac
 got=$("$SHELL" --posix -c '{ time; } 2>&1; times')
 [[ $got =~ $exp ]] || err_exit "POSIX time/times output: expected match of $(printf %q "$exp"), got $(printf %q "$got")"