Skip to content

Commit

Permalink
env: fix a bunch of escaping/parsing edge cases (#308)
Browse files Browse the repository at this point in the history 
* Handle standard escape sequences based on XSI/XBD
  https://pubs.opengroup.org/onlinepubs/9699919799/utilities/echo.html
* Fix parsing with escaped quotes at end of quoted
  value
* Handle key whose name starts with `export` (require
  whitespace between `export` keyword when stripping)
* Fix parsing of unquoted values with embedded `#`
  (require a space before starting an inline comment)
* Use correct parser methods throughout all test cases
* Eliminate a lot of unused code that was not working
  right

Signed-off-by: Milas Bowman <milas.bowman@docker.com>
  • Loading branch information
@milas
milas authored Sep 26, 2022
1 parent be0dada commit 08a21df
Show file tree
Hide file tree
Showing 5 changed files with 121 additions and 755 deletions.
134 changes: 21 additions & 113 deletions dotenv/godotenv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,17 @@
//
// The TL;DR is that you make a .env file that looks something like
//
// SOME_ENV_VAR=somevalue
// SOME_ENV_VAR=somevalue
//
// and then in your go code you can call
//
// godotenv.Load()
// godotenv.Load()
//
// and all the env vars declared in .env will be available through os.Getenv("SOME_ENV_VAR")
package dotenv

import (
"bytes"
"errors"
"fmt"
"io"
"os"
Expand All @@ -28,10 +27,10 @@ import (
"github.com/compose-spec/compose-go/template"
)

const doubleQuoteSpecialChars = "\\\n\r\"!$`"

var utf8BOM = []byte("\uFEFF")

var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored

// LookupFn represents a lookup function to resolve variables from
type LookupFn func(string) (string, bool)

Expand Down Expand Up @@ -60,13 +59,13 @@ func ParseWithLookup(r io.Reader, lookupFn LookupFn) (map[string]string, error)

// Load will read your env file(s) and load them into ENV for this process.
//
// Call this function as close as possible to the start of your program (ideally in main)
// Call this function as close as possible to the start of your program (ideally in main).
//
// If you call Load without any args it will default to loading .env in the current path
// If you call Load without any args it will default to loading .env in the current path.
//
// You can otherwise tell it which files to load (there can be more than one) like
// You can otherwise tell it which files to load (there can be more than one) like:
//
// godotenv.Load("fileone", "filetwo")
// godotenv.Load("fileone", "filetwo")
//
// It's important to note that it WILL NOT OVERRIDE an env variable that already exists - consider the .env file to set dev vars or sensible defaults
func Load(filenames ...string) error {
Expand All @@ -75,13 +74,13 @@ func Load(filenames ...string) error {

// Overload will read your env file(s) and load them into ENV for this process.
//
// Call this function as close as possible to the start of your program (ideally in main)
// Call this function as close as possible to the start of your program (ideally in main).
//
// If you call Overload without any args it will default to loading .env in the current path
// If you call Overload without any args it will default to loading .env in the current path.
//
// You can otherwise tell it which files to load (there can be more than one) like
// You can otherwise tell it which files to load (there can be more than one) like:
//
// godotenv.Overload("fileone", "filetwo")
// godotenv.Overload("fileone", "filetwo")
//
// It's important to note this WILL OVERRIDE an env variable that already exists - consider the .env file to forcefilly set all vars.
func Overload(filenames ...string) error {
Expand All @@ -99,8 +98,6 @@ func load(overload bool, filenames ...string) error {
return nil
}

var startsWithDigitRegex = regexp.MustCompile(`^\s*\d.*`) // Keys starting with numbers are ignored

// ReadWithLookup gets all env vars from the files and/or lookup function and return values as
// a map rather than automatically writing values into env
func ReadWithLookup(lookupFn LookupFn, filenames ...string) (map[string]string, error) {
Expand Down Expand Up @@ -155,6 +152,8 @@ func UnmarshalBytesWithLookup(src []byte, lookupFn LookupFn) (map[string]string,
//
// If you want more fine grained control over your command it's recommended
// that you use `Load()` or `Read()` and the `os/exec` package yourself.
//
// Deprecated: Use the `os/exec` package directly.
func Exec(filenames []string, cmd string, cmdArgs []string) error {
if err := Load(filenames...); err != nil {
return err
Expand All @@ -168,7 +167,10 @@ func Exec(filenames []string, cmd string, cmdArgs []string) error {
}

// Write serializes the given environment and writes it to a file
//
// Deprecated: The serialization functions are untested and unmaintained.
func Write(envMap map[string]string, filename string) error {
//goland:noinspection GoDeprecation
content, err := Marshal(envMap)
if err != nil {
return err
Expand All @@ -187,6 +189,8 @@ func Write(envMap map[string]string, filename string) error {

// Marshal outputs the given environment as a dotenv-formatted environment file.
// Each line is in the format: KEY="VALUE" where VALUE is backslash-escaped.
//
// Deprecated: The serialization functions are untested and unmaintained.
func Marshal(envMap map[string]string) (string, error) {
lines := make([]string, 0, len(envMap))
for k, v := range envMap {
Expand Down Expand Up @@ -239,104 +243,6 @@ func readFile(filename string, lookupFn LookupFn) (map[string]string, error) {
return ParseWithLookup(file, lookupFn)
}

var exportRegex = regexp.MustCompile(`^\s*(?:export\s+)?(.*?)\s*$`)

func parseLine(line string, envMap map[string]string) (string, string, error) {
return parseLineWithLookup(line, envMap, nil)
}
func parseLineWithLookup(line string, envMap map[string]string, lookupFn LookupFn) (string, string, error) {
if line == "" {
return "", "", errors.New("zero length string")
}

// ditch the comments (but keep quoted hashes)
if strings.HasPrefix(strings.TrimSpace(line), "#") || strings.Contains(line, " #") {
segmentsBetweenHashes := strings.Split(line, "#")
quotesAreOpen := false
var segmentsToKeep []string
for _, segment := range segmentsBetweenHashes {
if strings.Count(segment, "\"") == 1 || strings.Count(segment, "'") == 1 {
if quotesAreOpen {
segmentsToKeep = append(segmentsToKeep, segment)
}
quotesAreOpen = !quotesAreOpen
}

if len(segmentsToKeep) == 0 || quotesAreOpen {
segmentsToKeep = append(segmentsToKeep, segment)
}
}

line = strings.Join(segmentsToKeep, "#")
}

firstEquals := strings.Index(line, "=")
firstColon := strings.Index(line, ":")
splitString := strings.SplitN(line, "=", 2)
if firstColon != -1 && (firstColon < firstEquals || firstEquals == -1) {
// This is a yaml-style line
splitString = strings.SplitN(line, ":", 2)
}

if len(splitString) != 2 {
return "", "", errors.New("can't separate key from value")
}
key := exportRegex.ReplaceAllString(splitString[0], "$1")

// Parse the value
value := parseValue(splitString[1], envMap, lookupFn)

return key, value, nil
}

var (
singleQuotesRegex = regexp.MustCompile(`\A'(.*)'\z`)
doubleQuotesRegex = regexp.MustCompile(`\A"(.*)"\z`)
escapeRegex = regexp.MustCompile(`\\.`)
unescapeCharsRegex = regexp.MustCompile(`\\([^$])`)
)

func parseValue(value string, envMap map[string]string, lookupFn LookupFn) string {

// trim
value = strings.Trim(value, " ")

// check if we've got quoted values or possible escapes
if len(value) > 1 {
singleQuotes := singleQuotesRegex.FindStringSubmatch(value)

doubleQuotes := doubleQuotesRegex.FindStringSubmatch(value)

if singleQuotes != nil || doubleQuotes != nil {
// pull the quotes off the edges
value = value[1 : len(value)-1]
}

if doubleQuotes != nil {
// expand newlines
value = escapeRegex.ReplaceAllStringFunc(value, func(match string) string {
c := strings.TrimPrefix(match, `\`)
switch c {
case "n":
return "\n"
case "r":
return "\r"
default:
return match
}
})
// unescape characters
value = unescapeCharsRegex.ReplaceAllString(value, "$1")
}

if singleQuotes == nil {
value, _ = expandVariables(value, envMap, lookupFn)
}
}

return value
}

func expandVariables(value string, envMap map[string]string, lookupFn LookupFn) (string, error) {
retVal, err := template.Substitute(value, func(k string) (string, bool) {
if v, ok := envMap[k]; ok {
Expand All @@ -350,7 +256,9 @@ func expandVariables(value string, envMap map[string]string, lookupFn LookupFn)
return retVal, nil
}

// Deprecated: only used by unsupported/untested code for Marshal/Write.
func doubleQuoteEscape(line string) string {
const doubleQuoteSpecialChars = "\\\n\r\"!$`"
for _, c := range doubleQuoteSpecialChars {
toReplace := "\\" + string(c)
if c == '\n' {
Expand Down
Loading

0 comments on commit 08a21df

Please sign in to comment.