ash: add process substitution in bash-compatibility mode

Process substitution is a Korn shell feature that's also available
in bash and some other shells.  This patch implements process
substitution in ash when ASH_BASH_COMPAT is enabled.

function                                             old     new   delta
argstr                                              1386    1522    +136
strtodest                                              -      52     +52
readtoken1                                          3346    3392     +46
.rodata                                           183206  183250     +44
unwindredir                                            -      28     +28
cmdloop                                              365     372      +7
static.spclchars                                      10      12      +2
cmdputs                                              380     367     -13
exitreset                                             86      69     -17
evalcommand                                         1754    1737     -17
varvalue                                             675     634     -41
------------------------------------------------------------------------------
(add/remove: 2/0 grow/shrink: 5/4 up/down: 315/-88)           Total: 227 bytes
   text	   data	    bss	    dec	    hex	filename
 953967	   4219	   1904	 960090	  ea65a	busybox_old
 954192	   4219	   1904	 960315	  ea73b	busybox_unstripped

v2: Replace array of file descriptors with a linked list.
    Include tests that were unaccountably omitted from v1.
v3: Update linked list code to the intended version.
v4: Change order of conditional code in cmdputs().
v5: Use existing popredir() mechanism to manage file descriptors.
v6: Rebase to latest version of BusyBox ash.  Reduce code churn.

Signed-off-by: Ron Yorston <rmy@pobox.com>
Signed-off-by: Denys Vlasenko <vda.linux@googlemail.com>

include/platform.h

@@ -426,6 +426,8 @@ typedef unsigned smalluint;
 #define HAVE_SYS_STATFS_H 1
 #define HAVE_PRINTF_PERCENTM 1
 #define HAVE_WAIT3 1
+#define HAVE_DEV_FD 1
+#define DEV_FD_PREFIX "/dev/fd/"
 
 #if defined(__UCLIBC__)
 # if UCLIBC_VERSION < KERNEL_VERSION(0, 9, 32)

shell/ash.c

@@ -229,6 +229,14 @@
 #define    BASH_READ_D          ENABLE_ASH_BASH_COMPAT
 #define IF_BASH_READ_D              IF_ASH_BASH_COMPAT
 #define    BASH_WAIT_N          ENABLE_ASH_BASH_COMPAT
+/* <(...) and >(...) */
+#if HAVE_DEV_FD
+# define    BASH_PROCESS_SUBST   ENABLE_ASH_BASH_COMPAT
+# define IF_BASH_PROCESS_SUBST       IF_ASH_BASH_COMPAT
+#else
+# define    BASH_PROCESS_SUBST 0
+# define IF_BASH_PROCESS_SUBST(...)
+#endif
 
 #if defined(__ANDROID_API__) && __ANDROID_API__ <= 24
 /* Bionic at least up to version 24 has no glob() */
@@ -804,6 +812,12 @@ out2str(const char *p)
 #define CTLENDARI    ((unsigned char)'\207')
 #define CTLQUOTEMARK ((unsigned char)'\210')
 #define CTL_LAST CTLQUOTEMARK
+#if BASH_PROCESS_SUBST
+# define CTLTOPROC    ((unsigned char)'\211')
+# define CTLFROMPROC  ((unsigned char)'\212')
+# undef CTL_LAST
+# define CTL_LAST CTLFROMPROC
+#endif
 
 /* variable substitution byte (follows CTLVAR) */
 #define VSTYPE  0x0f            /* type of variable substitution */
@@ -1075,6 +1089,10 @@ trace_puts_quoted(char *s)
 		case CTLESC: c = 'e'; goto backslash;
 		case CTLVAR: c = 'v'; goto backslash;
 		case CTLBACKQ: c = 'q'; goto backslash;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC: c = 'p'; goto backslash;
+		case CTLFROMPROC: c = 'P'; goto backslash;
+#endif
  backslash:
 			putc('\\', tracefile);
 			putc(c, tracefile);
@@ -1236,8 +1254,17 @@ sharg(union node *arg, FILE *fp)
 		case CTLENDVAR:
 			putc('}', fp);
 			break;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+			putc('>', fp);
+			goto backq;
+		case CTLFROMPROC:
+			putc('<', fp);
+			goto backq;
+#endif
 		case CTLBACKQ:
 			putc('$', fp);
+ IF_BASH_PROCESS_SUBST(backq:)
 			putc('(', fp);
 			shtree(bqlist->n, -1, NULL, fp);
 			putc(')', fp);
@@ -3234,8 +3261,13 @@ static const uint8_t syntax_index_table[] ALIGN1 = {
 	/* 134 CTLARI       */ CCTL_CCTL_CCTL_CCTL,
 	/* 135 CTLENDARI    */ CCTL_CCTL_CCTL_CCTL,
 	/* 136 CTLQUOTEMARK */ CCTL_CCTL_CCTL_CCTL,
+#if BASH_PROCESS_SUBST
+	/* 137 CTLTOPROC    */ CCTL_CCTL_CCTL_CCTL,
+	/* 138 CTLFROMPROC  */ CCTL_CCTL_CCTL_CCTL,
+#else
 	/* 137      */ CWORD_CWORD_CWORD_CWORD,
 	/* 138      */ CWORD_CWORD_CWORD_CWORD,
+#endif
 	/* 139      */ CWORD_CWORD_CWORD_CWORD,
 	/* 140      */ CWORD_CWORD_CWORD_CWORD,
 	/* 141      */ CWORD_CWORD_CWORD_CWORD,
@@ -4849,9 +4881,24 @@ cmdputs(const char *s)
 			quoted >>= 1;
 			subtype = 0;
 			goto dostr;
+#if BASH_PROCESS_SUBST
+		case CTLBACKQ:
+			c = '$';
+			str = "(...)";
+			break;
+		case CTLTOPROC:
+			c = '>';
+			str = "(...)";
+			break;
+		case CTLFROMPROC:
+			c = '<';
+			str = "(...)";
+			break;
+#else
 		case CTLBACKQ:
 			str = "$(...)";
 			goto dostr;
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		case CTLARI:
 			str = "$((";
@@ -5891,6 +5938,21 @@ redirectsafe(union node *redir, int flags)
 	return err;
 }
 
+#if BASH_PROCESS_SUBST
+static void
+pushfd(int fd)
+{
+	struct redirtab *sv;
+
+	sv = ckzalloc(sizeof(*sv) + sizeof(sv->two_fd[0]));
+	sv->pair_count = 1;
+	sv->two_fd[0].orig_fd = fd;
+	sv->two_fd[0].moved_to = CLOSED;
+	sv->next = redirlist;
+	redirlist = sv;
+}
+#endif
+
 static struct redirtab*
 pushredir(union node *redir)
 {
@@ -6529,10 +6591,20 @@ evaltreenr(union node *n, int flags)
 }
 
 static void FAST_FUNC
-evalbackcmd(union node *n, struct backcmd *result)
+evalbackcmd(union node *n, struct backcmd *result
+				IF_BASH_PROCESS_SUBST(, int ctl))
 {
 	int pip[2];
 	struct job *jp;
+#if BASH_PROCESS_SUBST
+	/* determine end of pipe used by parent (ip) and child (ic) */
+	const int ip = (ctl == CTLTOPROC);
+	const int ic = !(ctl == CTLTOPROC);
+#else
+	const int ctl = CTLBACKQ;
+	const int ip = 0;
+	const int ic = 1;
+#endif
 
 	result->fd = -1;
 	result->buf = NULL;
@@ -6544,15 +6616,17 @@ evalbackcmd(union node *n, struct backcmd *result)
 
 	if (pipe(pip) < 0)
 		ash_msg_and_raise_perror("can't create pipe");
-	jp = makejob(/*n,*/ 1);
+	/* process substitution uses NULL job/node, like openhere() */
-	if (forkshell(jp, n, FORK_NOJOB) == 0) {
+	jp = (ctl == CTLBACKQ) ? makejob(/*n,*/ 1) : NULL;
+	if (forkshell(jp, (ctl == CTLBACKQ) ? n : NULL, FORK_NOJOB) == 0) {
 		/* child */
 		FORCE_INT_ON;
-		close(pip[0]);
+		close(pip[ip]);
-		if (pip[1] != 1) {
+		/* ic is index of child end of pipe *and* fd to connect it to */
-			/*close(1);*/
+		if (pip[ic] != ic) {
-			dup2_or_raise(pip[1], 1);
+			/*close(ic);*/
-			close(pip[1]);
+			dup2_or_raise(pip[ic], ic);
+			close(pip[ic]);
 		}
 /* TODO: eflag clearing makes the following not abort:
  *  ash -c 'set -e; z=$(false;echo foo); echo $z'
@@ -6568,8 +6642,18 @@ evalbackcmd(union node *n, struct backcmd *result)
 		/* NOTREACHED */
 	}
 	/* parent */
-	close(pip[1]);
+#if BASH_PROCESS_SUBST
-	result->fd = pip[0];
+	if (ctl != CTLBACKQ) {
+		int fd = fcntl(pip[ip], F_DUPFD, 64);
+		if (fd > 0) {
+			close(pip[ip]);
+			pip[ip] = fd;
+		}
+		pushfd(pip[ip]);
+	}
+#endif
+	close(pip[ic]);
+	result->fd = pip[ip];
 	result->jp = jp;
 
  out:
@@ -6581,8 +6665,11 @@ evalbackcmd(union node *n, struct backcmd *result)
  * Expand stuff in backwards quotes.
  */
 static void
-expbackq(union node *cmd, int flag)
+expbackq(union node *cmd, int flag IF_BASH_PROCESS_SUBST(, int ctl))
 {
+#if !BASH_PROCESS_SUBST
+	const int ctl = CTLBACKQ;
+#endif
 	struct backcmd in;
 	int i;
 	char buf[128];
@@ -6597,9 +6684,15 @@ expbackq(union node *cmd, int flag)
 	INT_OFF;
 	startloc = expdest - (char *)stackblock();
 	pushstackmark(&smark, startloc);
-	evalbackcmd(cmd, &in);
+	evalbackcmd(cmd, &in IF_BASH_PROCESS_SUBST(, ctl));
 	popstackmark(&smark);
 
+	if (ctl != CTLBACKQ) {
+		sprintf(buf, DEV_FD_PREFIX"%d", in.fd);
+		strtodest(buf, BASESYNTAX);
+		goto done;
+	}
+
 	p = in.buf;
 	i = in.nleft;
 	if (i == 0)
@@ -6621,6 +6714,7 @@ expbackq(union node *cmd, int flag)
 		close(in.fd);
 		back_exitstatus = waitforjob(in.jp);
 	}
+ done:
 	INT_ON;
 
 	/* Eat all trailing newlines */
@@ -6708,6 +6802,10 @@ argstr(char *p, int flag)
 		CTLESC,
 		CTLVAR,
 		CTLBACKQ,
+#if BASH_PROCESS_SUBST
+		CTLTOPROC,
+		CTLFROMPROC,
+#endif
 #if ENABLE_FEATURE_SH_MATH
 		CTLARI,
 		CTLENDARI,
@@ -6807,8 +6905,12 @@ argstr(char *p, int flag)
 			p = evalvar(p, flag | inquotes);
 			TRACE(("argstr: evalvar:'%s'\n", (char *)stackblock()));
 			goto start;
+#if BASH_PROCESS_SUBST
+		case CTLTOPROC:
+		case CTLFROMPROC:
+#endif
 		case CTLBACKQ:
-			expbackq(argbackq->n, flag | inquotes);
+			expbackq(argbackq->n, flag | inquotes IF_BASH_PROCESS_SUBST(, c));
 			goto start;
 #if ENABLE_FEATURE_SH_MATH
 		case CTLARI:
@@ -12198,8 +12300,9 @@ realeofmark(const char *eofmark)
 #define CHECKEND()      {goto checkend; checkend_return:;}
 #define PARSEREDIR()    {goto parseredir; parseredir_return:;}
 #define PARSESUB()      {goto parsesub; parsesub_return:;}
-#define PARSEBACKQOLD() {oldstyle = 1; goto parsebackq; parsebackq_oldreturn:;}
+#define PARSEBACKQOLD() {style = OLD; goto parsebackq; parsebackq_oldreturn:;}
-#define PARSEBACKQNEW() {oldstyle = 0; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEBACKQNEW() {style = NEW; goto parsebackq; parsebackq_newreturn:;}
+#define PARSEPROCSUB()  {style = PSUB; goto parsebackq; parsebackq_psreturn:;}
 #define PARSEARITH()    {goto parsearith; parsearith_return:;}
 static int
 readtoken1(int c, int syntax, char *eofmark, int striptabs)
@@ -12210,7 +12313,9 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 	size_t len;
 	struct nodelist *bqlist;
 	smallint quotef;
-	smallint oldstyle;
+	smallint style;
+	enum { OLD, NEW, PSUB };
+#define oldstyle (style == OLD)
 	smallint pssyntax;   /* we are expanding a prompt string */
 	IF_BASH_DOLLAR_SQUOTE(smallint bash_dollar_squote = 0;)
 	/* syntax stack */
@@ -12391,6 +12496,15 @@ readtoken1(int c, int syntax, char *eofmark, int striptabs)
 						c = 0x100 + '>'; /* flag &> */
 					pungetc();
 				}
+#endif
+#if BASH_PROCESS_SUBST
+				if (c == '<' || c == '>') {
+					if (pgetc() == '(') {
+						PARSEPROCSUB();
+						break;
+					}
+					pungetc();
+				}
 #endif
 				goto endword;   /* exit outer loop */
 			}
@@ -12876,9 +12990,18 @@ parsebackq: {
 		memcpy(out, str, savelen);
 		STADJUST(savelen, out);
 	}
-	USTPUTC(CTLBACKQ, out);
+#if BASH_PROCESS_SUBST
+	if (style == PSUB)
+		USTPUTC(c == '<' ? CTLFROMPROC : CTLTOPROC, out);
+	else
+#endif
+		USTPUTC(CTLBACKQ, out);
 	if (oldstyle)
 		goto parsebackq_oldreturn;
+#if BASH_PROCESS_SUBST
+	else if (style == PSUB)
+		goto parsebackq_psreturn;
+#endif
 	goto parsebackq_newreturn;
 }
 
@@ -13329,6 +13452,9 @@ cmdloop(int top)
 #if JOBS
 		if (doing_jobctl)
 			showjobs(SHOW_CHANGED|SHOW_STDERR);
+#endif
+#if BASH_PROCESS_SUBST
+		unwindredir(NULL);
 #endif
 		inter = 0;
 		if (iflag && top) {

shell/ash_test/ash-psubst/bash_procsub.right

@@ -0,0 +1,9 @@
+hello 1
+hello 2
+hello 3
+<(echo "hello 0")
+hello 4
+HI THERE
+hello error
+hello error
+hello stderr

shell/ash_test/ash-psubst/bash_procsub.tests

@@ -0,0 +1,33 @@
+# simplest case
+cat <(echo "hello 1")
+
+# can have more than one
+cat <(echo "hello 2") <(echo "hello 3")
+
+# doesn't work in quotes
+echo "<(echo \"hello 0\")"
+
+# process substitution can be nested inside command substitution
+echo $(cat <(echo "hello 4"))
+
+# example from http://wiki.bash-hackers.org/syntax/expansion/proc_subst
+# process substitutions can be passed to a function as parameters or
+# variables
+f() {
+	cat "$1" >"$x"
+}
+x=>(tr '[:lower:]' '[:upper:]') f <(echo 'hi there')
+
+# process substitution can be combined with redirection on exec
+rm -f err
+# save stderr
+exec 4>&2
+# copy stderr to a file
+exec 2> >(tee err)
+echo "hello error" >&2
+sync
+# restore stderr
+exec 2>&4
+cat err
+rm -f err
+echo "hello stderr" >&2