// Copyright (c) 2017, Daniel Martí // See LICENSE for licensing information package interp import ( "bytes" "context" "errors" "fmt" "io" "io/fs" "iter" "math" mathrand "math/rand/v2" "os" "path/filepath" "regexp" "runtime" "slices" "strconv" "strings" "sync" "time" "mvdan.cc/sh/v3/expand" "mvdan.cc/sh/v3/pattern" "mvdan.cc/sh/v3/syntax" ) const ( // shellReplyPS3Var, or PS3, is a special variable in Bash used by the select command, // while the shell is awaiting for input. the default value is [shellDefaultPS3] shellReplyPS3Var = "PS3" // shellDefaultPS3, or #?, is PS3's default value shellDefaultPS3 = "#? " // shellReplyVar, or REPLY, is a special variable in Bash that is used to store the result of // the select command or of the read command, when no variable name is specified shellReplyVar = "REPLY" fifoNamePrefix = "sh-interp-" ) func (r *Runner) fillExpandConfig(ctx context.Context) { r.ectx = ctx r.ecfg = &expand.Config{ Env: expandEnv{r}, CmdSubst: func(w io.Writer, cs *syntax.CmdSubst) error { switch len(cs.Stmts) { case 0: // nothing to do return nil case 1: // $( 100 { return "", fmt.Errorf("giving up at creating fifo: %v", err) } } r2 := r.subshell(true) stdout := r.origStdout // TODO: note that `man bash` mentions that `wait` only waits for the last // process substitution as long as it is $!; the logic here would mean we wait for all of them. bg := bgProc{ done: make(chan struct{}), exit: new(exitStatus), } r.bgProcs = append(r.bgProcs, bg) go func() { defer func() { *bg.exit = r2.exit close(bg.done) }() switch ps.Op { case syntax.CmdIn: f, err := os.OpenFile(path, os.O_WRONLY, 0) if err != nil { r.errf("cannot open fifo for stdout: %v\n", err) return } r2.stdout = f defer func() { if err := f.Close(); err != nil { r.errf("closing stdout fifo: %v\n", err) } os.Remove(path) }() default: // syntax.CmdOut f, err := os.OpenFile(path, os.O_RDONLY, 0) if err != nil { r.errf("cannot open fifo for stdin: %v\n", err) return } r2.stdin = f r2.stdout = stdout defer func() { f.Close() os.Remove(path) }() } r2.stmts(ctx, ps.Stmts) r2.exit.exiting = false // subshells don't exit the parent shell }() return path, nil }, } r.updateExpandOpts() } // catShortcutArg checks if a statement is of the form "$(, while , etc) // part of && or || lists; excluded via "else" above // preceded by !; excluded via "else" above if r.opts[optErrExit] { r.exit.exiting = true } } if !r.keepRedirs { r.stdin, r.stdout, r.stderr = oldIn, oldOut, oldErr } } func (r *Runner) cmd(ctx context.Context, cm syntax.Command) { if r.stop(ctx) { return } tracingEnabled := r.opts[optXTrace] trace := r.tracer() switch cm := cm.(type) { case *syntax.Block: r.stmts(ctx, cm.Stmts) case *syntax.Subshell: r2 := r.subshell(false) r2.stmts(ctx, cm.Stmts) r2.exit.exiting = false // subshells don't exit the parent shell r.exit = r2.exit case *syntax.CallExpr: // Use a new slice, to not modify the slice in the alias map. args := cm.Args for i := 0; i < len(args); { if !r.opts[optExpandAliases] { break } als, ok := r.alias[args[i].Lit()] if !ok { break } args = slices.Replace(args, i, i+1, als.args...) if !als.blank { break } i += len(als.args) } r.lastExpandExit = exitStatus{} fields := r.fields(args...) if len(fields) == 0 { for _, as := range cm.Assigns { prev := r.lookupVar(as.Name.Value) // Here we have a naked "foo=bar", so if we inherited a local var from a parent // function we want to signal that we are modifying the parent var rather than // creating a new local var via "local foo=bar". // TODO: there is likely a better way to do this. prev.Local = false vr := r.assignVal(prev, as, "") r.setVarWithIndex(prev, as.Name.Value, as.Index, vr) if !tracingEnabled { continue } // Strangely enough, it seems like Bash prints original // source for arrays, but the expanded value otherwise. // TODO: add test cases for x[i]=y and x+=y. if as.Array != nil { trace.expr(as) } else if as.Value != nil { val, err := syntax.Quote(vr.String(), syntax.LangBash) if err != nil { // should never happen panic(err) } trace.stringf("%s=%s", as.Name.Value, val) } trace.newLineFlush() } // If interpreting the last expansion like $(foo) failed, // and the expansion and assignments otherwise succeeded, // we need to surface that last exit code. if r.exit.ok() { r.exit = r.lastExpandExit } break } type restoreVar struct { name string vr expand.Variable } var restores []restoreVar for _, as := range cm.Assigns { name := as.Name.Value prev := r.lookupVar(name) vr := r.assignVal(prev, as, "") // Inline command vars are always exported. vr.Exported = true restores = append(restores, restoreVar{name, prev}) r.setVar(name, vr) } trace.call(fields[0], fields[1:]...) trace.newLineFlush() r.call(ctx, cm.Args[0].Pos(), fields) for _, restore := range restores { r.setVar(restore.name, restore.vr) } case *syntax.BinaryCmd: switch cm.Op { case syntax.AndStmt, syntax.OrStmt: oldNoErrExit := r.noErrExit r.noErrExit = true r.stmt(ctx, cm.X) r.noErrExit = oldNoErrExit if r.exit.ok() == (cm.Op == syntax.AndStmt) { r.stmt(ctx, cm.Y) } case syntax.Pipe, syntax.PipeAll: pr, pw, err := os.Pipe() if err != nil { r.exit.fatal(err) // not being able to create a pipe is rare but critical return } r2 := r.subshell(true) r2.stdout = pw if cm.Op == syntax.PipeAll { r2.stderr = pw } else { r2.stderr = r.stderr } r.stdin = pr var wg sync.WaitGroup wg.Add(1) go func() { r2.stmt(ctx, cm.X) r2.exit.exiting = false // subshells don't exit the parent shell pw.Close() wg.Done() }() r.stmt(ctx, cm.Y) pr.Close() wg.Wait() if r.opts[optPipeFail] && !r2.exit.ok() && r.exit.ok() { r.exit = r2.exit } if r2.exit.fatalExit { r.exit.fatal(r2.exit.err) // surface fatal errors immediately } } case *syntax.IfClause: oldNoErrExit := r.noErrExit r.noErrExit = true r.stmts(ctx, cm.Cond) r.noErrExit = oldNoErrExit if r.exit.ok() { r.stmts(ctx, cm.Then) break } r.exit.code = 0 if cm.Else != nil { r.cmd(ctx, cm.Else) } case *syntax.WhileClause: for !r.stop(ctx) { oldNoErrExit := r.noErrExit r.noErrExit = true r.stmts(ctx, cm.Cond) r.noErrExit = oldNoErrExit stop := r.exit.ok() == cm.Until r.exit.code = 0 if stop || r.loopStmtsBroken(ctx, cm.Do) { break } } case *syntax.ForClause: switch y := cm.Loop.(type) { case *syntax.WordIter: name := y.Name.Value items := r.Params // for i; do ... inToken := y.InPos.IsValid() if inToken { items = r.fields(y.Items...) // for i in ...; do ... } if cm.Select { ps3 := shellDefaultPS3 if e := r.envGet(shellReplyPS3Var); e != "" { ps3 = e } prompt := func() []byte { // display menu for i, word := range items { r.errf("%d) %v\n", i+1, word) } r.errf("%s", ps3) line, err := r.readLine(ctx, true) if err != nil { r.exit.code = 1 return nil } return line } retry: choice := prompt() if len(choice) == 0 { goto retry // no reply; try again } reply := string(choice) r.setVarString(shellReplyVar, reply) c, _ := strconv.Atoi(reply) if c > 0 && c <= len(items) { r.setVarString(name, items[c-1]) } // execute commands until break or return is encountered if r.loopStmtsBroken(ctx, cm.Do) { break } } for _, field := range items { r.setVarString(name, field) trace.stringf("for %s in", y.Name.Value) if inToken { for _, item := range y.Items { trace.string(" ") trace.expr(item) } } else { trace.string(` "$@"`) } trace.newLineFlush() if r.loopStmtsBroken(ctx, cm.Do) { break } } case *syntax.CStyleLoop: if y.Init != nil { r.arithm(y.Init) } for y.Cond == nil || r.arithm(y.Cond) != 0 { if !r.exit.ok() || r.loopStmtsBroken(ctx, cm.Do) { break } if y.Post != nil { r.arithm(y.Post) } } } case *syntax.FuncDecl: r.setFunc(cm.Name.Value, cm.Body) case *syntax.ArithmCmd: r.exit.oneIf(r.arithm(cm.X) == 0) case *syntax.LetClause: var val int for _, expr := range cm.Exprs { val = r.arithm(expr) if !tracingEnabled { continue } switch expr := expr.(type) { case *syntax.Word: qs, err := syntax.Quote(r.literal(expr), syntax.LangBash) if err != nil { return } trace.stringf("let %v", qs) case *syntax.BinaryArithm, *syntax.UnaryArithm: trace.expr(cm) case *syntax.ParenArithm: // TODO } } trace.newLineFlush() r.exit.oneIf(val == 0) case *syntax.CaseClause: trace.string("case ") trace.expr(cm.Word) trace.string(" in") trace.newLineFlush() str := r.literal(cm.Word) for _, ci := range cm.Items { for _, word := range ci.Patterns { pattern := r.pattern(word) if match(pattern, str) { r.stmts(ctx, ci.Stmts) return } } } case *syntax.TestClause: if r.bashTest(ctx, cm.X, false) == "" && r.exit.ok() { // to preserve exit status code 2 for regex errors, etc r.exit.code = 1 } case *syntax.DeclClause: local, global := false, false var modes []string valType := "" switch cm.Variant.Value { case "declare": // When used in a function, "declare" acts as "local" // unless the "-g" option is used. local = r.inFunc case "local": if !r.inFunc { r.errf("local: can only be used in a function\n") r.exit.code = 1 return } local = true case "export": modes = append(modes, "-x") case "readonly": modes = append(modes, "-r") case "nameref": valType = "-n" } assignLoop: for as := range r.flattenAssigns(cm.Args) { fp := flagParser{remaining: []string{as.Name.Value}} for fp.more() { switch flag := fp.flag(); flag { case "-x", "-r": modes = append(modes, flag) case "-a", "-A", "-n": valType = flag case "-g": global = true default: r.errf("declare: invalid option %q\n", flag) r.exit.code = 2 return } continue assignLoop } name := as.Name.Value if !syntax.ValidName(name) { r.errf("declare: invalid name %q\n", name) r.exit.code = 1 return } vr := r.lookupVar(as.Name.Value) if as.Naked { if valType == "-A" { vr.Kind = expand.Associative } else { vr.Kind = expand.KeepValue } } else { vr = r.assignVal(vr, as, valType) } if global { vr.Local = false } else if local { vr.Local = true } for _, mode := range modes { switch mode { case "-x": vr.Exported = true case "-r": vr.ReadOnly = true } } r.setVar(name, vr) } case *syntax.TimeClause: start := time.Now() if cm.Stmt != nil { r.stmt(ctx, cm.Stmt) } format := "%s\t%s\n" if cm.PosixFormat { format = "%s %s\n" } else { r.outf("\n") } real := time.Since(start) r.outf(format, "real", elapsedString(real, cm.PosixFormat)) // TODO: can we do these? r.outf(format, "user", elapsedString(0, cm.PosixFormat)) r.outf(format, "sys", elapsedString(0, cm.PosixFormat)) default: panic(fmt.Sprintf("unhandled command node: %T", cm)) } } func (r *Runner) trapCallback(ctx context.Context, callback, name string) { if callback == "" { return // nothing to do } if r.handlingTrap { return // don't recurse, as that could lead to cycles } r.handlingTrap = true p := syntax.NewParser() // TODO: do this parsing when "trap" is called? file, err := p.Parse(strings.NewReader(callback), name+" trap") if err != nil { r.errf(name+"trap: %v\n", err) // ignore errors in the callback return } oldExit := r.exit r.stmts(ctx, file.Stmts) r.exit = oldExit // traps on EXIT or ERR should not modify the result r.handlingTrap = false } func (r *Runner) flattenAssigns(args []*syntax.Assign) iter.Seq[*syntax.Assign] { return func(yield func(*syntax.Assign) bool) { for _, as := range args { // Convert "declare $x" into "declare value". // Don't use syntax.Parser here, as we only want the basic // splitting by '='. if as.Name != nil { if !yield(as) { return } continue } for _, field := range r.fields(as.Value) { as := &syntax.Assign{} name, val, ok := strings.Cut(field, "=") as.Name = &syntax.Lit{Value: name} if !ok { as.Naked = true } else { as.Value = &syntax.Word{Parts: []syntax.WordPart{ &syntax.Lit{Value: val}, }} } if !yield(as) { return } } } } } func match(pat, name string) bool { expr, err := pattern.Regexp(pat, pattern.EntireString) if err != nil { return false } rx := regexp.MustCompile(expr) return rx.MatchString(name) } func elapsedString(d time.Duration, posix bool) string { if posix { return fmt.Sprintf("%.2f", d.Seconds()) } min := int(d.Minutes()) sec := math.Mod(d.Seconds(), 60.0) return fmt.Sprintf("%dm%.3fs", min, sec) } func (r *Runner) stmts(ctx context.Context, stmts []*syntax.Stmt) { for _, stmt := range stmts { r.stmt(ctx, stmt) } } func (r *Runner) hdocReader(rd *syntax.Redirect) (*os.File, error) { pr, pw, err := os.Pipe() if err != nil { return nil, err } // We write to the pipe in a new goroutine, // as pipe writes may block once the buffer gets full. // We still construct and buffer the entire heredoc first, // as doing it concurrently would lead to different semantics and be racy. if rd.Op != syntax.DashHdoc { hdoc := r.document(rd.Hdoc) go func() { pw.WriteString(hdoc) pw.Close() }() return pr, nil } var buf bytes.Buffer var cur []syntax.WordPart flushLine := func() { if buf.Len() > 0 { buf.WriteByte('\n') } buf.WriteString(r.document(&syntax.Word{Parts: cur})) cur = cur[:0] } for _, wp := range rd.Hdoc.Parts { lit, ok := wp.(*syntax.Lit) if !ok { cur = append(cur, wp) continue } for i, part := range strings.Split(lit.Value, "\n") { if i > 0 { flushLine() cur = cur[:0] } part = strings.TrimLeft(part, "\t") cur = append(cur, &syntax.Lit{Value: part}) } } flushLine() go func() { pw.Write(buf.Bytes()) pw.Close() }() return pr, nil } func (r *Runner) redir(ctx context.Context, rd *syntax.Redirect) (io.Closer, error) { if rd.Hdoc != nil { pr, err := r.hdocReader(rd) if err != nil { return nil, err } r.stdin = pr return pr, nil } orig := &r.stdout if rd.N != nil { switch rd.N.Value { case "0": // Note that the input redirects below always use stdin (0) // because we don't support anything else right now. case "1": // The default for the output redirects below. case "2": orig = &r.stderr default: panic(fmt.Sprintf("unsupported redirect fd: %v", rd.N.Value)) } } arg := r.literal(rd.Word) switch rd.Op { case syntax.WordHdoc: pr, pw, err := os.Pipe() if err != nil { return nil, err } r.stdin = pr // We write to the pipe in a new goroutine, // as pipe writes may block once the buffer gets full. go func() { pw.WriteString(arg) pw.WriteString("\n") pw.Close() }() return pr, nil case syntax.DplOut: switch arg { case "1": *orig = r.stdout case "2": *orig = r.stderr case "-": *orig = io.Discard // closing the output writer default: panic(fmt.Sprintf("unhandled %v arg: %q", rd.Op, arg)) } return nil, nil case syntax.RdrIn, syntax.RdrOut, syntax.AppOut, syntax.RdrAll, syntax.AppAll: // done further below case syntax.DplIn: switch arg { case "-": r.stdin = nil // closing the input file default: panic(fmt.Sprintf("unhandled %v arg: %q", rd.Op, arg)) } return nil, nil default: panic(fmt.Sprintf("unhandled redirect op: %v", rd.Op)) } mode := os.O_RDONLY switch rd.Op { case syntax.AppOut, syntax.AppAll: mode = os.O_WRONLY | os.O_CREATE | os.O_APPEND case syntax.RdrOut, syntax.RdrAll: mode = os.O_WRONLY | os.O_CREATE | os.O_TRUNC } f, err := r.open(ctx, arg, mode, 0o644, true) if err != nil { return nil, err } switch rd.Op { case syntax.RdrIn: stdin, err := stdinFile(f) if err != nil { return nil, err } r.stdin = stdin case syntax.RdrOut, syntax.AppOut: *orig = f case syntax.RdrAll, syntax.AppAll: r.stdout = f r.stderr = f default: panic(fmt.Sprintf("unhandled redirect op: %v", rd.Op)) } return f, nil } func (r *Runner) loopStmtsBroken(ctx context.Context, stmts []*syntax.Stmt) bool { oldInLoop := r.inLoop r.inLoop = true defer func() { r.inLoop = oldInLoop }() for _, stmt := range stmts { r.stmt(ctx, stmt) if r.contnEnclosing > 0 { r.contnEnclosing-- return r.contnEnclosing > 0 } if r.breakEnclosing > 0 { r.breakEnclosing-- return true } } return false } func (r *Runner) call(ctx context.Context, pos syntax.Pos, args []string) { if r.stop(ctx) { return } if r.callHandler != nil { var err error args, err = r.callHandler(r.handlerCtx(ctx, handlerKindCall, pos), args) if err != nil { // handler's custom fatal error r.exit.fatal(err) return } } name := args[0] if body := r.Funcs[name]; body != nil { // stack them to support nested func calls oldParams := r.Params r.Params = args[1:] oldInFunc := r.inFunc r.inFunc = true // Functions run in a nested scope. // Note that [Runner.exec] below does something similar. origEnv := r.writeEnv r.writeEnv = &overlayEnviron{parent: r.writeEnv, funcScope: true} r.stmt(ctx, body) r.writeEnv = origEnv r.Params = oldParams r.inFunc = oldInFunc r.exit.returning = false return } if IsBuiltin(name) { r.exit = r.builtin(ctx, pos, name, args[1:]) return } r.exec(ctx, pos, args) } func (r *Runner) exec(ctx context.Context, pos syntax.Pos, args []string) { r.exit.fromHandlerError(r.execHandler(r.handlerCtx(ctx, handlerKindExec, pos), args)) } func (r *Runner) open(ctx context.Context, path string, flags int, mode os.FileMode, print bool) (io.ReadWriteCloser, error) { // If we are opening a FIFO temporary file created by the interpreter itself, // don't pass this along to the open handler as it will not work at all // unless [os.OpenFile] is used directly with it. // Matching by directory and basename prefix isn't perfect, but works. // // If we want FIFOs to use a handler in the future, they probably // need their own separate handler API matching Unix-like semantics. dir, name := filepath.Split(path) dir = strings.TrimSuffix(dir, "/") if dir == r.tempDir && strings.HasPrefix(name, fifoNamePrefix) { return os.OpenFile(path, flags, mode) } f, err := r.openHandler(r.handlerCtx(ctx, handlerKindOpen, todoPos), path, flags, mode) // TODO: support wrapped PathError returned from openHandler. switch err.(type) { case nil: return f, nil case *os.PathError: if print { r.errf("%v\n", err) } default: // handler's custom fatal error r.exit.fatal(err) } return nil, err } func (r *Runner) stat(ctx context.Context, name string) (fs.FileInfo, error) { path := absPath(r.Dir, name) return r.statHandler(ctx, path, true) } func (r *Runner) lstat(ctx context.Context, name string) (fs.FileInfo, error) { path := absPath(r.Dir, name) return r.statHandler(ctx, path, false) }