489 lines
13 KiB
Go
489 lines
13 KiB
Go
// Package goawk is an implementation of AWK with CSV support
|
|
//
|
|
// You can use the command-line "goawk" command or run AWK from your
|
|
// Go programs using the "interp" package. The command-line program
|
|
// has the same interface as regular awk:
|
|
//
|
|
// goawk [-F fs] [-v var=value] [-f progfile | 'prog'] [file ...]
|
|
//
|
|
// The -F flag specifies the field separator (the default is to split
|
|
// on whitespace). The -v flag allows you to set a variable to a
|
|
// given value (multiple -v flags allowed). The -f flag allows you to
|
|
// read AWK source from a file instead of the 'prog' command-line
|
|
// argument. The rest of the arguments are input filenames (default
|
|
// is to read from stdin).
|
|
//
|
|
// A simple example (prints the sum of the numbers in the file's
|
|
// second column):
|
|
//
|
|
// $ echo 'foo 12
|
|
// > bar 34
|
|
// > baz 56' >file.txt
|
|
// $ goawk '{ sum += $2 } END { print sum }' file.txt
|
|
// 102
|
|
//
|
|
// To use GoAWK in your Go programs, see README.md or the "interp"
|
|
// package docs.
|
|
package main
|
|
|
|
import (
|
|
"bytes"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"runtime/pprof"
|
|
"strings"
|
|
"unicode/utf8"
|
|
|
|
"github.com/benhoyt/goawk/internal/compiler"
|
|
"github.com/benhoyt/goawk/internal/cover"
|
|
"github.com/benhoyt/goawk/internal/parseutil"
|
|
"github.com/benhoyt/goawk/internal/resolver"
|
|
"github.com/benhoyt/goawk/interp"
|
|
"github.com/benhoyt/goawk/lexer"
|
|
"github.com/benhoyt/goawk/parser"
|
|
)
|
|
|
|
const (
|
|
version = "v1.31.0"
|
|
copyright = "GoAWK " + version + " - Copyright (c) 2022 Ben Hoyt"
|
|
shortUsage = "usage: goawk [-F fs] [-v var=value] [-f progfile | 'prog'] [file ...]"
|
|
longUsage = `Standard AWK arguments:
|
|
--csv enable CSV input mode; equivalent to "-i csv"
|
|
-F separator field separator (default " ")
|
|
-f progfile load AWK source from progfile (multiple allowed)
|
|
-v var=value variable assignment (multiple allowed)
|
|
|
|
Additional GoAWK features:
|
|
-c use Unicode chars for index, length, match, substr, and %c
|
|
-E progfile load program, treat as last option, disable var=value args
|
|
-H parse header row and enable @"field" in CSV input mode
|
|
-h, --help show this help message
|
|
-i mode parse input into fields using CSV format (ignore FS and RS)
|
|
'csv|tsv [separator=<char>] [comment=<char>] [header]'
|
|
-o mode use CSV output for print with args (ignore OFS and ORS)
|
|
'csv|tsv [separator=<char>]'
|
|
-N mode newline output translation: smart (default), raw, crlf
|
|
-version show GoAWK version and exit
|
|
|
|
GoAWK debugging arguments:
|
|
-coverappend append to coverage profile instead of overwriting
|
|
-covermode mode set coverage mode: set, count (default "set")
|
|
-coverprofile fn write coverage profile to file
|
|
-cpuprofile fn write CPU profile to file
|
|
-d print parsed syntax tree to stdout and exit
|
|
-da print VM assembly instructions to stdout and exit
|
|
-dt print variable type information to stdout and exit
|
|
-memprofile fn write memory profile to file
|
|
`
|
|
)
|
|
|
|
func main() {
|
|
// Parse command line arguments manually rather than using the
|
|
// "flag" package, so we can support flags with no space between
|
|
// flag and argument, like '-F:' (allowed by POSIX)
|
|
var progFiles []string
|
|
var vars []string
|
|
fieldSep := " "
|
|
cpuProfile := ""
|
|
debug := false
|
|
debugAsm := false
|
|
debugTypes := false
|
|
memProfile := ""
|
|
inputMode := ""
|
|
outputMode := ""
|
|
header := false
|
|
noArgVars := false
|
|
coverMode := cover.ModeUnspecified
|
|
coverProfile := ""
|
|
coverAppend := false
|
|
useChars := false
|
|
newlineOutput := interp.SmartNewlineMode
|
|
|
|
var i int
|
|
argsLoop:
|
|
for i = 1; i < len(os.Args); i++ {
|
|
// Stop on explicit end of args or first arg not prefixed with "-"
|
|
arg := os.Args[i]
|
|
if arg == "--" {
|
|
i++
|
|
break
|
|
}
|
|
if arg == "-" || !strings.HasPrefix(arg, "-") {
|
|
break
|
|
}
|
|
|
|
switch arg {
|
|
case "-covermode":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -covermode")
|
|
}
|
|
i++
|
|
coverMode = coverModeFromString(os.Args[i])
|
|
case "-coverprofile":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -coverprofile")
|
|
}
|
|
i++
|
|
coverProfile = os.Args[i]
|
|
case "-coverappend":
|
|
coverAppend = true
|
|
case "-E":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -E")
|
|
}
|
|
i++
|
|
progFiles = append(progFiles, os.Args[i])
|
|
noArgVars = true
|
|
i++
|
|
break argsLoop
|
|
case "-F":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -F")
|
|
}
|
|
i++
|
|
fieldSep = os.Args[i]
|
|
case "-f":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -f")
|
|
}
|
|
i++
|
|
progFiles = append(progFiles, os.Args[i])
|
|
case "-v":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -v")
|
|
}
|
|
i++
|
|
vars = append(vars, os.Args[i])
|
|
case "-cpuprofile":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -cpuprofile")
|
|
}
|
|
i++
|
|
cpuProfile = os.Args[i]
|
|
case "-csv", "--csv":
|
|
inputMode = "csv"
|
|
case "-c":
|
|
useChars = true
|
|
case "-d":
|
|
debug = true
|
|
case "-da":
|
|
debugAsm = true
|
|
case "-dt":
|
|
debugTypes = true
|
|
case "-H":
|
|
header = true
|
|
case "-h", "--help":
|
|
fmt.Printf("%s\n\n%s\n\n%s", copyright, shortUsage, longUsage)
|
|
os.Exit(0)
|
|
case "-i":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -i")
|
|
}
|
|
i++
|
|
inputMode = os.Args[i]
|
|
case "-memprofile":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -memprofile")
|
|
}
|
|
i++
|
|
memProfile = os.Args[i]
|
|
case "-o":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -o")
|
|
}
|
|
i++
|
|
outputMode = os.Args[i]
|
|
case "-N":
|
|
if i+1 >= len(os.Args) {
|
|
errorExitf("flag needs an argument: -N")
|
|
}
|
|
i++
|
|
newlineOutput = newlineModeFromString(os.Args[i])
|
|
case "-version", "--version":
|
|
fmt.Println(version)
|
|
os.Exit(0)
|
|
default:
|
|
switch {
|
|
case strings.HasPrefix(arg, "-E"):
|
|
progFiles = append(progFiles, arg[2:])
|
|
noArgVars = true
|
|
i++
|
|
break argsLoop
|
|
case strings.HasPrefix(arg, "-F"):
|
|
fieldSep = arg[2:]
|
|
case strings.HasPrefix(arg, "-f"):
|
|
progFiles = append(progFiles, arg[2:])
|
|
case strings.HasPrefix(arg, "-i"):
|
|
inputMode = arg[2:]
|
|
case strings.HasPrefix(arg, "-o"):
|
|
outputMode = arg[2:]
|
|
case strings.HasPrefix(arg, "-v"):
|
|
vars = append(vars, arg[2:])
|
|
case strings.HasPrefix(arg, "-N"):
|
|
newlineOutput = newlineModeFromString(arg[2:])
|
|
case strings.HasPrefix(arg, "-cpuprofile="):
|
|
cpuProfile = arg[len("-cpuprofile="):]
|
|
case strings.HasPrefix(arg, "-memprofile="):
|
|
memProfile = arg[len("-memprofile="):]
|
|
case strings.HasPrefix(arg, "-covermode="):
|
|
coverMode = coverModeFromString(arg[len("-covermode="):])
|
|
case strings.HasPrefix(arg, "-coverprofile="):
|
|
coverProfile = arg[len("-coverprofile="):]
|
|
default:
|
|
errorExitf("flag provided but not defined: %s", arg)
|
|
}
|
|
}
|
|
}
|
|
if coverProfile != "" && coverMode == cover.ModeUnspecified {
|
|
coverMode = cover.ModeSet
|
|
}
|
|
|
|
// Any remaining args are program and input files
|
|
args := os.Args[i:]
|
|
|
|
fileReader := &parseutil.FileReader{}
|
|
if len(progFiles) > 0 {
|
|
// Read source: the concatenation of all source files specified
|
|
progFiles = expandWildcardsOnWindows(progFiles)
|
|
for _, progFile := range progFiles {
|
|
if progFile == "-" {
|
|
err := fileReader.AddFile("<stdin>", os.Stdin)
|
|
if err != nil {
|
|
errorExit(err)
|
|
}
|
|
} else {
|
|
f, err := os.Open(progFile)
|
|
if err != nil {
|
|
errorExit(err)
|
|
}
|
|
err = fileReader.AddFile(progFile, f)
|
|
if err != nil {
|
|
_ = f.Close()
|
|
errorExit(err)
|
|
}
|
|
_ = f.Close()
|
|
}
|
|
}
|
|
} else {
|
|
if len(args) < 1 {
|
|
errorExitf(shortUsage)
|
|
}
|
|
err := fileReader.AddFile("<cmdline>", strings.NewReader(args[0]))
|
|
if err != nil {
|
|
errorExit(err)
|
|
}
|
|
args = args[1:]
|
|
}
|
|
|
|
// Parse source code and setup interpreter
|
|
parserConfig := &parser.ParserConfig{
|
|
DebugTypes: debugTypes,
|
|
DebugWriter: os.Stdout,
|
|
}
|
|
prog, err := parser.ParseProgram(fileReader.Source(), parserConfig)
|
|
if err != nil {
|
|
if err, ok := err.(*parser.ParseError); ok {
|
|
name, line := fileReader.FileLine(err.Position.Line)
|
|
fmt.Fprintf(os.Stderr, "%s:%d:%d: %s\n",
|
|
name, line, err.Position.Column, err.Message)
|
|
showSourceLine(fileReader.Source(), err.Position)
|
|
os.Exit(1)
|
|
}
|
|
errorExitf("%s", err)
|
|
}
|
|
|
|
coverage := cover.New(coverMode, coverAppend, fileReader)
|
|
|
|
if coverMode != cover.ModeUnspecified {
|
|
astProgram := &prog.ResolvedProgram.Program
|
|
coverage.Annotate(astProgram)
|
|
|
|
// re-resolve annotated program
|
|
prog.ResolvedProgram = *resolver.Resolve(astProgram, &resolver.Config{
|
|
DebugTypes: parserConfig.DebugTypes,
|
|
DebugWriter: parserConfig.DebugWriter})
|
|
|
|
// re-compile it
|
|
prog.Compiled, err = compiler.Compile(&prog.ResolvedProgram)
|
|
if err != nil {
|
|
errorExitf("%s", err)
|
|
}
|
|
}
|
|
|
|
if debug {
|
|
fmt.Fprintln(os.Stdout, prog)
|
|
}
|
|
|
|
if debugAsm {
|
|
err := prog.Disassemble(os.Stdout)
|
|
if err != nil {
|
|
errorExitf("could not disassemble program: %v", err)
|
|
}
|
|
}
|
|
|
|
if debug || debugAsm || debugTypes {
|
|
os.Exit(0)
|
|
}
|
|
|
|
if header {
|
|
if inputMode == "" {
|
|
errorExitf("-H only allowed together with -i")
|
|
}
|
|
inputMode += " header"
|
|
}
|
|
|
|
// Don't buffer output if stdout is a terminal (default output writer when
|
|
// Config.Output is nil is a buffered version of os.Stdout).
|
|
var stdout io.Writer
|
|
stdoutInfo, err := os.Stdout.Stat()
|
|
if err == nil && stdoutInfo.Mode()&os.ModeCharDevice != 0 {
|
|
stdout = os.Stdout
|
|
}
|
|
|
|
config := &interp.Config{
|
|
Argv0: filepath.Base(os.Args[0]),
|
|
Args: expandWildcardsOnWindows(args),
|
|
Chars: useChars,
|
|
NoArgVars: noArgVars,
|
|
Output: stdout,
|
|
NewlineOutput: newlineOutput,
|
|
Vars: []string{
|
|
"FS", fieldSep,
|
|
"INPUTMODE", inputMode,
|
|
"OUTPUTMODE", outputMode,
|
|
},
|
|
}
|
|
for _, v := range vars {
|
|
name, value, ok := strings.Cut(v, "=")
|
|
if !ok {
|
|
errorExitf("-v flag must be in format name=value")
|
|
}
|
|
// Oddly, -v must interpret escapes (issue #129)
|
|
unescaped, err := lexer.Unescape(value)
|
|
if err == nil {
|
|
value = unescaped
|
|
}
|
|
config.Vars = append(config.Vars, name, value)
|
|
}
|
|
|
|
if cpuProfile != "" {
|
|
f, err := os.Create(cpuProfile)
|
|
if err != nil {
|
|
errorExitf("could not create CPU profile: %v", err)
|
|
}
|
|
if err := pprof.StartCPUProfile(f); err != nil {
|
|
errorExitf("could not start CPU profile: %v", err)
|
|
}
|
|
}
|
|
|
|
// Run the program!
|
|
interpreter, err := interp.New(prog)
|
|
if err != nil {
|
|
errorExit(err)
|
|
}
|
|
status, err := interpreter.Execute(config)
|
|
if err != nil {
|
|
errorExit(err)
|
|
}
|
|
|
|
if coverProfile != "" {
|
|
err := coverage.WriteProfile(coverProfile, interpreter.Array(cover.ArrayName))
|
|
if err != nil {
|
|
errorExitf("unable to write coverage profile: %v", err)
|
|
}
|
|
}
|
|
|
|
if cpuProfile != "" {
|
|
pprof.StopCPUProfile()
|
|
}
|
|
if memProfile != "" {
|
|
f, err := os.Create(memProfile)
|
|
if err != nil {
|
|
errorExitf("could not create memory profile: %v", err)
|
|
}
|
|
runtime.GC() // get up-to-date statistics
|
|
if err := pprof.WriteHeapProfile(f); err != nil {
|
|
errorExitf("could not write memory profile: %v", err)
|
|
}
|
|
_ = f.Close()
|
|
}
|
|
|
|
os.Exit(status)
|
|
}
|
|
|
|
func newlineModeFromString(mode string) interp.NewlineMode {
|
|
switch mode {
|
|
case "smart":
|
|
return interp.SmartNewlineMode
|
|
case "raw":
|
|
return interp.RawNewlineMode
|
|
case "crlf":
|
|
return interp.CRLFNewlineMode
|
|
default:
|
|
errorExitf("-N arg can only be one of: smart, raw, crlf")
|
|
return interp.SmartNewlineMode
|
|
}
|
|
}
|
|
|
|
func coverModeFromString(mode string) cover.Mode {
|
|
switch mode {
|
|
case "set":
|
|
return cover.ModeSet
|
|
case "count":
|
|
return cover.ModeCount
|
|
default:
|
|
errorExitf("-covermode can only be one of: set, count")
|
|
return cover.ModeUnspecified
|
|
}
|
|
}
|
|
|
|
// Show source line and position of error, for example:
|
|
//
|
|
// BEGIN { x*; }
|
|
// ^
|
|
func showSourceLine(src []byte, pos lexer.Position) {
|
|
lines := bytes.Split(src, []byte{'\n'})
|
|
srcLine := string(lines[pos.Line-1])
|
|
numTabs := strings.Count(srcLine[:pos.Column-1], "\t")
|
|
runeColumn := utf8.RuneCountInString(srcLine[:pos.Column-1])
|
|
fmt.Fprintln(os.Stderr, strings.ReplaceAll(srcLine, "\t", " "))
|
|
fmt.Fprintln(os.Stderr, strings.Repeat(" ", runeColumn)+strings.Repeat(" ", numTabs)+"^")
|
|
}
|
|
|
|
func errorExit(err error) {
|
|
pathErr, ok := err.(*os.PathError)
|
|
if ok && os.IsNotExist(err) {
|
|
errorExitf("file %q not found", pathErr.Path)
|
|
}
|
|
errorExitf("%s", err)
|
|
}
|
|
|
|
func errorExitf(format string, args ...any) {
|
|
fmt.Fprintf(os.Stderr, format+"\n", args...)
|
|
os.Exit(1)
|
|
}
|
|
|
|
func expandWildcardsOnWindows(args []string) []string {
|
|
if runtime.GOOS != "windows" {
|
|
return args
|
|
}
|
|
return expandWildcards(args)
|
|
}
|
|
|
|
// Originally from https://github.com/mattn/getwild (compatible LICENSE).
|
|
func expandWildcards(args []string) []string {
|
|
result := make([]string, 0, len(args))
|
|
for _, arg := range args {
|
|
matches, err := filepath.Glob(arg)
|
|
if err == nil && len(matches) > 0 {
|
|
result = append(result, matches...)
|
|
} else {
|
|
result = append(result, arg)
|
|
}
|
|
}
|
|
return result
|
|
}
|