Fix ability to link 3rd party code against libshell (re: f2bc1f4)

Over the course of developing ksh, the three public ksh/libshell
interface headers (shell.h, nval.h, history.h), which should make
it possible to embed the shell in third party programs by linking
against libshell, got messed up so they are no longer suitable for
public interface use. This commit fixes that, restoring this
ability.

src/cmd/ksh93/Mamfile:
- Define _BLD_ksh=1 while building ksh.

src/cmd/ksh93/include/{shell,nval,history}.h:
- Compile out all non-public parts of the interface if _BLD_ksh is
  not defined.
- Similarly hide ann internal #include directive based on _BLD_ksh.
- Make most other checks for SHOPT_* shell options conditional upon
  _BLD_ksh being defined; if not, we're building a third-party
  program and the SHOPT_* macros aren't available, so just assume
  them to be on.

src/cmd/INIT/dylink.sh:
- While we're here, fix a minor omission in $prefix usage ('lib'
  was still hardcoded in one place). (re: 4d35de0)

bin/package, src/cmd/INIT/Mamfile:
- Set C_INCLUDE_PATH to $INSTALLROOT/include/ast for (at least) gcc
  and clang, allowing users to omit the corresponding -I flag while
  compiling against libast on the command line from the
  'bin/package use' environment.
- This invalidates some iffe regression tests that assume non-AST
  headers, so we have to unset it again before running those.
___________________________________________________________________
For reference, below is a little test program that demonstrates
some aspects of writing a libshell program in C.

Here are the manual pages needed to understand the program (the
'man' command will work with full path arguments on most systems):

- src/lib/libast/man/LIBAST.3
- src/lib/libast/man/sfio.3
- src/lib/libast/man/cdt.3
- src/cmd/ksh93/shell.3
- src/cmd/ksh93/nval.3

Save as test.c, then build and run from the test environment:

$ bin/package make -j3  # if not already done
$ bin/package use
$ cc test.c -l ast -l shell
$ ./a.out

===begin test.c===
    #include <shell.h>

    void myuserinit(Shell_t *shp, int reinit)
    {
        char *greet;
        NOT_USED(shp);
        if (reinit == 0)
            greet = "Hello";
        else if (reinit > 0)
            greet = "Hi again";
        else
            greet = "Bye";
        sfprintf(sfstdout, "%s! reinit==%d\n", greet, reinit);
    }

    int main(int argc, char *argv[])
    {
        Namval_t *np;

        sh_init(argc, argv, myuserinit);

        /* Write an introduction using a shell command */
        sh_trap("print \"This is libshell ${.sh.version}\"", 0);

        /* Walk through the variables table and print all the
         * pre-set and inherited variables */
        for(np = dtfirst(sh.var_tree); np;
            np = dtnext(sh.var_tree,np))
        {
                sfprintf(sfstdout, " * %s == %s\n",
                    nv_name(np), sh_fmtq(nv_getval(np)));
        }
        return 0;
    }
===end test.c===

bin/package

@@ -131,7 +131,7 @@ command=${0##*/}
 case $(getopts '[-][123:xyz]' opt --xyz 2>/dev/null; echo 0$opt) in
 0123)	USAGE=$'
 [-?
-@(#)$Id: '$command$' (ksh 93u+m) 2024-08-16 $
+@(#)$Id: '$command$' (ksh 93u+m) 2024-12-07 $
 ]
 [-author?Glenn Fowler <[email protected]>]
 [-author?Contributors to https://github.com/ksh93/ksh]
@@ -564,7 +564,7 @@ SEE ALSO
   pkgadd(1), pkgmk(1), rpm(1), sh(1), tar(1), optget(3)
 
 IMPLEMENTATION
-  version         package (ksh 93u+m) 2024-08-16
+  version         package (ksh 93u+m) 2024-12-07
   author          Glenn Fowler <[email protected]>
   author          Contributors to https://github.com/ksh93/ksh
   copyright       (c) 1994-2012 AT&T Intellectual Property
@@ -2080,7 +2080,7 @@ case $x in
 			$show _RLD_ROOT=$_RLD_ROOT
 			$show export _RLD_ROOT
 			export _RLD_ROOT
-			# Haiku
+			# dynamic library search path for gcc and clang; also used for runtime linking on Haiku
 			case $LIBRARY_PATH: in
 			$INSTALLROOT/dyn/lib:*)
 				;;
@@ -2090,6 +2090,16 @@ case $x in
 				export LIBRARY_PATH
 				;;
 			esac
+			# include header search path for gcc and clang
+			case $C_INCLUDE_PATH: in
+			$INSTALLROOT/include/ast:*)
+				;;
+			*)	C_INCLUDE_PATH=$INSTALLROOT/include/ast${C_INCLUDE_PATH:+:$C_INCLUDE_PATH}
+				$show C_INCLUDE_PATH=$C_INCLUDE_PATH
+				$show export C_INCLUDE_PATH
+				export C_INCLUDE_PATH
+				;;
+			esac
 			;;
 		esac
 

src/cmd/INIT/Mamfile

@@ -180,10 +180,11 @@ make test dontcare virtual
 	make test.mamake virtual
 		makp mamake.tst
 		exec - : testing non-libast mamake at $PWD/mamake :
-		exec - regress mamake.tst mamake
+		exec - regress --verbose %{<} mamake
 	done
 	make test.iffe virtual
 		makp iffe.tst
-		exec - regress iffe.tst iffe
+		exec - unset C_INCLUDE_PATH  # set by bin/package, but invalidates some iffe tests
+		exec - regress --verbose %{<} iffe
 	done
 done test

src/cmd/INIT/dylink.sh

@@ -123,7 +123,7 @@ case ${exec_file} in
 	esac
 	sym_links="$lib_linkname $prefix$module_name$suffix"
 	# ... remove possible old versions
-	(set +o noglob; exec rm -f "$dest_dir/lib/lib$module_name".*)
+	(set +o noglob; exec rm -f "$dest_dir/lib/$prefix$module_name".*)
 	# ... execute linker command
 	case $HOSTTYPE in
 	darwin.*)

src/cmd/ksh93/Mamfile

@@ -6,7 +6,7 @@ note *
 setv MAMAKE_STRICT 5
 setv INSTALLROOT ../../..
 setv CC cc
-setv mam_cc_FLAGS %{mam_cc_TARGET} %{mam_cc_DLL} %{-debug-symbols?1?%{mam_cc_DEBUG} -D_BLD_DEBUG?%{mam_cc_OPTIMIZE}?}
+setv mam_cc_FLAGS %{mam_cc_TARGET} %{mam_cc_DLL} -D_BLD_ksh %{-debug-symbols?1?%{mam_cc_DEBUG} -D_BLD_DEBUG?%{mam_cc_OPTIMIZE}?}
 setv CCFLAGS
 setv CCLDFLAGS %{-strip-symbols?1?%{mam_cc_LD_STRIP}??}
 setv IFFEFLAGS

src/cmd/ksh93/include/history.h

@@ -19,6 +19,8 @@
  *	Interface for history mechanism
  *	written by David Korn
  *
+ * NOTE: this header defines a public libshell interface,
+ * unless _BLD_ksh is defined as nonzero
  */
 
 #include	<ast.h>
@@ -82,7 +84,7 @@ extern int		hist_match(History_t*,off_t, char*, int*);
 extern off_t		hist_tell(History_t*,int);
 extern off_t		hist_seek(History_t*,int);
 extern char 		*hist_word(char*, int, int);
-#if SHOPT_ESH
+#if !_BLD_ksh || SHOPT_ESH
     extern Histloc_t	hist_locate(History_t*,int, int, int);
 #endif	/* SHOPT_ESH */
 

src/cmd/ksh93/include/nval.h

@@ -25,6 +25,8 @@
  * Interface definitions of structures for name-value pairs
  * These structures are used for named variables, functions and aliases
  *
+ * NOTE: this header defines a public libshell interface,
+ * unless _BLD_ksh is defined as nonzero
  */
 
 
@@ -134,11 +136,13 @@ struct Namval
 #define NV_REF		0x4000	/* reference bit */
 #define NV_TABLE	0x800	/* node is a dictionary table */
 #define NV_MINIMAL	0x1000	/* node does not contain all fields */
+#if _BLD_ksh
 #if SHOPT_OPTIMIZE
 #define NV_NOOPTIMIZE	NV_TABLE	/* disable loop invariants optimizer */
 #else
 #define NV_NOOPTIMIZE	0
-#endif
+#endif /* SHOPT_OPTIMIZE */
+#endif /* _BLD_ksh */
 
 #define NV_INTEGER	0x2	/* integer attribute */
 /* The following attributes are valid only when NV_INTEGER is off */

src/cmd/ksh93/include/shell.h

@@ -24,6 +24,8 @@
  *
  * Interface definitions for shell command language
  *
+ * NOTE: this header defines a public libshell interface,
+ * unless _BLD_ksh is defined as nonzero
  */
 
 #define SH_VERSION	20220719
@@ -32,12 +34,12 @@
 #include	<cdt.h>
 #include	<history.h>
 #include	<stk.h>
-#ifdef defs_h_defined
+#if _BLD_ksh
 #   include	"name.h"
+#   include	"fault.h"
 #else
 #   include	<nval.h>
-#endif /* defs_h_defined */
-#include	"fault.h"
+#endif /* _BLD_ksh */
 
 /* options */
 typedef struct
@@ -108,18 +110,18 @@ typedef union Shnode_u Shnode_t;
 #define SH_NOCLOBBER	14
 #define SH_MARKDIRS	15
 #define SH_BGNICE	16
-#if SHOPT_VSH
+#if !_BLD_ksh || SHOPT_VSH
 #define SH_VI		17
 #define SH_VIRAW	18
 #endif
 #define	SH_TFLAG	19
 #define SH_TRACKALL	20
 #define	SH_SFLAG	21
 #define	SH_NOEXEC	22
-#if SHOPT_ESH || SHOPT_VSH
+#if !_BLD_ksh || SHOPT_ESH || SHOPT_VSH
 #define SH_NOARROWSRCH	23
 #endif
-#if SHOPT_ESH
+#if !_BLD_ksh || SHOPT_ESH
 #define SH_GMACS	24
 #define SH_EMACS	25
 #endif
@@ -130,30 +132,33 @@ typedef union Shnode_u Shnode_t;
 #define SH_GLOBEX	31
 #define SH_PIPEFAIL	32
 #define SH_GLOBSTARS	33
-#if SHOPT_GLOBCASEDET || !defined(SHOPT_GLOBCASEDET)
+#if !_BLD_ksh || SHOPT_GLOBCASEDET || !defined(SHOPT_GLOBCASEDET)
 #define SH_GLOBCASEDET	34
 #endif
 #define SH_RC		35
 #define SH_SHOWME	36
 #define SH_LETOCTAL	37
-#if SHOPT_BRACEPAT
+#if !_BLD_ksh || SHOPT_BRACEPAT
 #define SH_BRACEEXPAND	42
 #endif
-#if SHOPT_HISTEXPAND
+#if !_BLD_ksh || SHOPT_HISTEXPAND
 #define SH_HISTEXPAND	43
-#if SHOPT_ESH || SHOPT_VSH
+#if !_BLD_ksh || SHOPT_ESH || SHOPT_VSH
 #define SH_HISTREEDIT	44
 #define SH_HISTVERIFY	45
 #endif
 #endif
 #define SH_POSIX	46
-#if SHOPT_ESH || SHOPT_VSH
+#if !_BLD_ksh || SHOPT_ESH || SHOPT_VSH
 #define SH_MULTILINE	47
 #define SH_NOBACKSLCTRL	48
 #endif
 #define SH_LOGIN_SHELL	67
+
+#if _BLD_ksh
 #define SH_NOUSRPROFILE	79	/* internal use only */
 #define SH_COMMANDLINE	0x100	/* bit flag for invocation-only options ('set -o' cannot change them) */
+#endif
 
 /*
  * passed as flags to builtins in Nambltin_t struct when BLT_OPTIM is on
@@ -188,6 +193,8 @@ typedef struct sh_scope
 	struct sh_scope	*self;
 } Shscope_t;
 
+#if _BLD_ksh
+
 /* Private interface to shell scope. The first members must match the public interface. */
 struct sh_scoped
 {
@@ -230,6 +237,8 @@ struct limits
 	int		child_max;	/* maximum number of children */
 };
 
+#endif /* _BLD_ksh */
+
 /*
  * Saves the state of the shell
  */
@@ -246,8 +255,8 @@ struct Shell_s
 	int		inlineno;	/* line number of current input file */
 	int		exitval;	/* exit status of the command currently being run */
 	int		savexit;	/* $? == exit status of the last command executed */
-
-	/* These are the former 'struct shared' (shgd) members. */
+#if _BLD_ksh
+	/* The rest are for ksh's internal use only */
 	struct limits	lim;
 	uid_t		userid;
 	uid_t		euserid;
@@ -268,12 +277,6 @@ struct Shell_s
 	void		*ed_context;
 	int		sigmax;
 	Shwait_f	waitevent;
-#if SHOPT_STATS
-	int		*stats;
-#endif
-
-	/* The following members are not considered to be part of the documented API.
-	 * Programs using libshell should not rely on them as they may change. */
 	int		subshell;	/* set for virtual subshell */
 	int		realsubshell;	/* ${.sh.subshell}, actual subshell level (including virtual and forked) */
 	char		nv_restore;	/* set while restoring variables upon terminating a virtual subshell */
@@ -393,6 +396,9 @@ struct Shell_s
 	char		nv_putsub_already_called_sh_arith;
 	int		nv_putsub_idx;	/* saves array index obtained by nv_putsub() using sh_arith() */
 	int16_t		level;		/* ${.sh.level} */
+#if SHOPT_STATS
+	int		*stats;
+#endif
 #if SHOPT_OPTIMIZE
 	char		**argaddr;	/* pointer to arguments for the loop invariants optimizer */
 	void		*optlist;	/* linked list of invariant nodes */
@@ -404,6 +410,7 @@ struct Shell_s
 	char		*fifo;		/* FIFO name for current process substitution */
 	Dt_t		*fifo_tree;	/* for cleaning up process substitution FIFOs */
 #endif /* !SHOPT_DEVFD */
+#endif /* _BLD_ksh */
 };
 
 /* used for builtins */

src/cmd/ksh93/shell.3

@@ -140,28 +140,25 @@ The function \fIfn\fP will be called with argument \f3-1\fP before
 the shell exits.
 .PD
 .PP
-The \f3Shell_t\fP structure contains the following fields:
+The \f3Shell_t\fP structure contains the following public members:
 .nf
 .ft 5
-	Shopt_t         \fIoptions\fP;        \fR/* set -o options */\fP
-	Dt_t            *\fIvar_tree\fP;      \fR/* shell variable dictionary */\fP
-	Dt_t            *\fIfun_tree\fP;      \fR/* shell function dictionary */\fP
-	Dt_t            *\fIalias_tree\fP;    \fR/* shell alias dictionary */\fP
-	Dt_t            *\fIbltin_tree\fP;    \fR/* shell built-in command dictionary */\fP
-	Dt_t            *\fItrack_tree\fP;    \fR/* shell hash table (tracked alias dictionary) */\fP
-	Shscope_t       *\fItopscope\fP;      \fR/* pointer to top-level scope */\fP
-	int             \fIinlineno\fP;       \fR/* line number of current input file */\fP
-	int             \fIexitval\fP;        \fR/* exit status of the command currently being run */\fP
-	int             \fIsavexit\fP;        \fR/* $? == exit status of the last command executed */\fP
+    Shopt_t   \fIoptions\fP;     \fR/* set -o options */\fP
+    Dt_t      *\fIvar_tree\fP;   \fR/* shell variable dictionary */\fP
+    Dt_t      *\fIfun_tree\fP;   \fR/* shell function dictionary */\fP
+    Dt_t      *\fIalias_tree\fP; \fR/* shell alias dictionary */\fP
+    Dt_t      *\fIbltin_tree\fP; \fR/* shell built-in command dictionary */\fP
+    Dt_t      *\fItrack_tree\fP; \fR/* shell hash table (tracked aliases) */\fP
+    Shscope_t *\fItopscope\fP;   \fR/* pointer to top-level scope */\fP
+    int       \fIinlineno\fP;    \fR/* line number of current input file */\fP
+    int       \fIexitval\fP;     \fR/* exit status of the current command */\fP
+    int       \fIsavexit\fP;     \fR/* exit status of the last command */\fP
 .ft R
 .fi
 As of ksh 93u+m, it is once again officially supported to access this
 structure directly as \f3sh\fP. In addition, a pointer to
 this structure is returned by \f3sh_init()\fP but can also be retrieved
 by a call to \f3sh_getinterp()\fP.
-There are other fields not documented here, but they should be considered
-unstable and subject to change between releases; programs using libshell
-should not rely on them.
 .PP
 All built-in commands to the shell are invoked with
 three arguments.  The first two arguments give the
@@ -315,11 +312,15 @@ provides an interface to some of the information that
 is available on each scope.  The structure contains
 the following public members:
 .nf
-      \f3Sh_scope_t      *par_scope;\fP
-      \f3int             argc;\fP
-      \f3char            **argv;\fP
-      \f3char            *cmdname;\fP
-      \f3Dt_t            *var_tree;\fP
+    Shscope_t *\fIpar_scope;\fP \fR/* pointer to parent scope */\fP
+    int       \fIargc;\fP       \fR/* number of arguments */\fP
+    char      **\fIargv;\fP     \fR/* array of arguments */\fP
+    char      *\fIcmdname;\fP   \fR/* command name */\fP
+    char      *\fIfilename;\fP  \fR/* file name */\fP
+    char      *\fIfunname;\fP   \fR/* function name */\fP
+    int       \fIlineno;\fP     \fR/* current script line number */\fP
+    Dt_t      *\fIvar_tree;\fP  \fR/* function's variables dictionary */\fP
+    Shscope_t *\fIself;\fP      \fR/* pointer to copy of this scope */\fP
 .fi
 .PP
 The \f3sh_getscope()\fP function can be used to get the