// Package goawk is an implementation of AWK with CSV support // // You can use the command-line "goawk" command or run AWK from your // Go programs using the "interp" package. The command-line program // has the same interface as regular awk: // // goawk [-F fs] [-v var=value] [-f progfile | 'prog'] [file ...] // // The -F flag specifies the field separator (the default is to split // on whitespace). The -v flag allows you to set a variable to a // given value (multiple -v flags allowed). The -f flag allows you to // read AWK source from a file instead of the 'prog' command-line // argument. The rest of the arguments are input filenames (default // is to read from stdin). // // A simple example (prints the sum of the numbers in the file's // second column): // // $ echo 'foo 12 // > bar 34 // > baz 56' >file.txt // $ goawk '{ sum += $2 } END { print sum }' file.txt // 102 // // To use GoAWK in your Go programs, see README.md or the "interp" // package docs. package main import ( "bytes" "fmt" "io" "os" "path/filepath" "runtime" "runtime/pprof" "strings" "unicode/utf8" "github.com/benhoyt/goawk/internal/compiler" "github.com/benhoyt/goawk/internal/cover" "github.com/benhoyt/goawk/internal/parseutil" "github.com/benhoyt/goawk/internal/resolver" "github.com/benhoyt/goawk/interp" "github.com/benhoyt/goawk/lexer" "github.com/benhoyt/goawk/parser" ) const ( version = "v1.31.0" copyright = "GoAWK " + version + " - Copyright (c) 2022 Ben Hoyt" shortUsage = "usage: goawk [-F fs] [-v var=value] [-f progfile | 'prog'] [file ...]" longUsage = `Standard AWK arguments: --csv enable CSV input mode; equivalent to "-i csv" -F separator field separator (default " ") -f progfile load AWK source from progfile (multiple allowed) -v var=value variable assignment (multiple allowed) Additional GoAWK features: -c use Unicode chars for index, length, match, substr, and %c -E progfile load program, treat as last option, disable var=value args -H parse header row and enable @"field" in CSV input mode -h, --help show this help message -i mode parse input into fields using CSV format (ignore FS and RS) 'csv|tsv [separator=] [comment=] [header]' -o mode use CSV output for print with args (ignore OFS and ORS) 'csv|tsv [separator=]' -N mode newline output translation: smart (default), raw, crlf -version show GoAWK version and exit GoAWK debugging arguments: -coverappend append to coverage profile instead of overwriting -covermode mode set coverage mode: set, count (default "set") -coverprofile fn write coverage profile to file -cpuprofile fn write CPU profile to file -d print parsed syntax tree to stdout and exit -da print VM assembly instructions to stdout and exit -dt print variable type information to stdout and exit -memprofile fn write memory profile to file ` ) func main() { // Parse command line arguments manually rather than using the // "flag" package, so we can support flags with no space between // flag and argument, like '-F:' (allowed by POSIX) var progFiles []string var vars []string fieldSep := " " cpuProfile := "" debug := false debugAsm := false debugTypes := false memProfile := "" inputMode := "" outputMode := "" header := false noArgVars := false coverMode := cover.ModeUnspecified coverProfile := "" coverAppend := false useChars := false newlineOutput := interp.SmartNewlineMode var i int argsLoop: for i = 1; i < len(os.Args); i++ { // Stop on explicit end of args or first arg not prefixed with "-" arg := os.Args[i] if arg == "--" { i++ break } if arg == "-" || !strings.HasPrefix(arg, "-") { break } switch arg { case "-covermode": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -covermode") } i++ coverMode = coverModeFromString(os.Args[i]) case "-coverprofile": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -coverprofile") } i++ coverProfile = os.Args[i] case "-coverappend": coverAppend = true case "-E": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -E") } i++ progFiles = append(progFiles, os.Args[i]) noArgVars = true i++ break argsLoop case "-F": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -F") } i++ fieldSep = os.Args[i] case "-f": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -f") } i++ progFiles = append(progFiles, os.Args[i]) case "-v": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -v") } i++ vars = append(vars, os.Args[i]) case "-cpuprofile": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -cpuprofile") } i++ cpuProfile = os.Args[i] case "-csv", "--csv": inputMode = "csv" case "-c": useChars = true case "-d": debug = true case "-da": debugAsm = true case "-dt": debugTypes = true case "-H": header = true case "-h", "--help": fmt.Printf("%s\n\n%s\n\n%s", copyright, shortUsage, longUsage) os.Exit(0) case "-i": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -i") } i++ inputMode = os.Args[i] case "-memprofile": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -memprofile") } i++ memProfile = os.Args[i] case "-o": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -o") } i++ outputMode = os.Args[i] case "-N": if i+1 >= len(os.Args) { errorExitf("flag needs an argument: -N") } i++ newlineOutput = newlineModeFromString(os.Args[i]) case "-version", "--version": fmt.Println(version) os.Exit(0) default: switch { case strings.HasPrefix(arg, "-E"): progFiles = append(progFiles, arg[2:]) noArgVars = true i++ break argsLoop case strings.HasPrefix(arg, "-F"): fieldSep = arg[2:] case strings.HasPrefix(arg, "-f"): progFiles = append(progFiles, arg[2:]) case strings.HasPrefix(arg, "-i"): inputMode = arg[2:] case strings.HasPrefix(arg, "-o"): outputMode = arg[2:] case strings.HasPrefix(arg, "-v"): vars = append(vars, arg[2:]) case strings.HasPrefix(arg, "-N"): newlineOutput = newlineModeFromString(arg[2:]) case strings.HasPrefix(arg, "-cpuprofile="): cpuProfile = arg[len("-cpuprofile="):] case strings.HasPrefix(arg, "-memprofile="): memProfile = arg[len("-memprofile="):] case strings.HasPrefix(arg, "-covermode="): coverMode = coverModeFromString(arg[len("-covermode="):]) case strings.HasPrefix(arg, "-coverprofile="): coverProfile = arg[len("-coverprofile="):] default: errorExitf("flag provided but not defined: %s", arg) } } } if coverProfile != "" && coverMode == cover.ModeUnspecified { coverMode = cover.ModeSet } // Any remaining args are program and input files args := os.Args[i:] fileReader := &parseutil.FileReader{} if len(progFiles) > 0 { // Read source: the concatenation of all source files specified progFiles = expandWildcardsOnWindows(progFiles) for _, progFile := range progFiles { if progFile == "-" { err := fileReader.AddFile("", os.Stdin) if err != nil { errorExit(err) } } else { f, err := os.Open(progFile) if err != nil { errorExit(err) } err = fileReader.AddFile(progFile, f) if err != nil { _ = f.Close() errorExit(err) } _ = f.Close() } } } else { if len(args) < 1 { errorExitf(shortUsage) } err := fileReader.AddFile("", strings.NewReader(args[0])) if err != nil { errorExit(err) } args = args[1:] } // Parse source code and setup interpreter parserConfig := &parser.ParserConfig{ DebugTypes: debugTypes, DebugWriter: os.Stdout, } prog, err := parser.ParseProgram(fileReader.Source(), parserConfig) if err != nil { if err, ok := err.(*parser.ParseError); ok { name, line := fileReader.FileLine(err.Position.Line) fmt.Fprintf(os.Stderr, "%s:%d:%d: %s\n", name, line, err.Position.Column, err.Message) showSourceLine(fileReader.Source(), err.Position) os.Exit(1) } errorExitf("%s", err) } coverage := cover.New(coverMode, coverAppend, fileReader) if coverMode != cover.ModeUnspecified { astProgram := &prog.ResolvedProgram.Program coverage.Annotate(astProgram) // re-resolve annotated program prog.ResolvedProgram = *resolver.Resolve(astProgram, &resolver.Config{ DebugTypes: parserConfig.DebugTypes, DebugWriter: parserConfig.DebugWriter}) // re-compile it prog.Compiled, err = compiler.Compile(&prog.ResolvedProgram) if err != nil { errorExitf("%s", err) } } if debug { fmt.Fprintln(os.Stdout, prog) } if debugAsm { err := prog.Disassemble(os.Stdout) if err != nil { errorExitf("could not disassemble program: %v", err) } } if debug || debugAsm || debugTypes { os.Exit(0) } if header { if inputMode == "" { errorExitf("-H only allowed together with -i") } inputMode += " header" } // Don't buffer output if stdout is a terminal (default output writer when // Config.Output is nil is a buffered version of os.Stdout). var stdout io.Writer stdoutInfo, err := os.Stdout.Stat() if err == nil && stdoutInfo.Mode()&os.ModeCharDevice != 0 { stdout = os.Stdout } config := &interp.Config{ Argv0: filepath.Base(os.Args[0]), Args: expandWildcardsOnWindows(args), Chars: useChars, NoArgVars: noArgVars, Output: stdout, NewlineOutput: newlineOutput, Vars: []string{ "FS", fieldSep, "INPUTMODE", inputMode, "OUTPUTMODE", outputMode, }, } for _, v := range vars { name, value, ok := strings.Cut(v, "=") if !ok { errorExitf("-v flag must be in format name=value") } // Oddly, -v must interpret escapes (issue #129) unescaped, err := lexer.Unescape(value) if err == nil { value = unescaped } config.Vars = append(config.Vars, name, value) } if cpuProfile != "" { f, err := os.Create(cpuProfile) if err != nil { errorExitf("could not create CPU profile: %v", err) } if err := pprof.StartCPUProfile(f); err != nil { errorExitf("could not start CPU profile: %v", err) } } // Run the program! interpreter, err := interp.New(prog) if err != nil { errorExit(err) } status, err := interpreter.Execute(config) if err != nil { errorExit(err) } if coverProfile != "" { err := coverage.WriteProfile(coverProfile, interpreter.Array(cover.ArrayName)) if err != nil { errorExitf("unable to write coverage profile: %v", err) } } if cpuProfile != "" { pprof.StopCPUProfile() } if memProfile != "" { f, err := os.Create(memProfile) if err != nil { errorExitf("could not create memory profile: %v", err) } runtime.GC() // get up-to-date statistics if err := pprof.WriteHeapProfile(f); err != nil { errorExitf("could not write memory profile: %v", err) } _ = f.Close() } os.Exit(status) } func newlineModeFromString(mode string) interp.NewlineMode { switch mode { case "smart": return interp.SmartNewlineMode case "raw": return interp.RawNewlineMode case "crlf": return interp.CRLFNewlineMode default: errorExitf("-N arg can only be one of: smart, raw, crlf") return interp.SmartNewlineMode } } func coverModeFromString(mode string) cover.Mode { switch mode { case "set": return cover.ModeSet case "count": return cover.ModeCount default: errorExitf("-covermode can only be one of: set, count") return cover.ModeUnspecified } } // Show source line and position of error, for example: // // BEGIN { x*; } // ^ func showSourceLine(src []byte, pos lexer.Position) { lines := bytes.Split(src, []byte{'\n'}) srcLine := string(lines[pos.Line-1]) numTabs := strings.Count(srcLine[:pos.Column-1], "\t") runeColumn := utf8.RuneCountInString(srcLine[:pos.Column-1]) fmt.Fprintln(os.Stderr, strings.ReplaceAll(srcLine, "\t", " ")) fmt.Fprintln(os.Stderr, strings.Repeat(" ", runeColumn)+strings.Repeat(" ", numTabs)+"^") } func errorExit(err error) { pathErr, ok := err.(*os.PathError) if ok && os.IsNotExist(err) { errorExitf("file %q not found", pathErr.Path) } errorExitf("%s", err) } func errorExitf(format string, args ...any) { fmt.Fprintf(os.Stderr, format+"\n", args...) os.Exit(1) } func expandWildcardsOnWindows(args []string) []string { if runtime.GOOS != "windows" { return args } return expandWildcards(args) } // Originally from https://github.com/mattn/getwild (compatible LICENSE). func expandWildcards(args []string) []string { result := make([]string, 0, len(args)) for _, arg := range args { matches, err := filepath.Glob(arg) if err == nil && len(matches) > 0 { result = append(result, matches...) } else { result = append(result, arg) } } return result }