Merge pull request #4427 from tstenner/variable_substitution

String substitution in variable expansion

frontend/dockerfile/docs/reference.md

@@ -357,6 +357,20 @@ directive in your Dockerfile:
   string=foobarbaz echo ${string%%b*}   # foo
   ```
 
+- `${variable/pattern/replacement}` replace the first occurrence of `pattern`
+  in `variable` with `replacement`
+
+  ```bash
+  string=foobarbaz echo ${string/ba/fo}  # fooforbaz
+  ```
+
+- `${variable//pattern/replacement}` replaces all occurrences of `pattern`
+  in `variable` with `replacement`
+
+  ```bash
+  string=foobarbaz echo ${string//ba/fo}  # fooforfoz
+  ```
+
 In all cases, `word` can be any string, including additional environment
 variables.
 

frontend/dockerfile/shell/lex.go

@@ -419,6 +419,45 @@ func (sw *shellWord) processDollar() (string, error) {
 		default:
 			return "", errors.Errorf("unsupported modifier (%s) in substitution", chs)
 		}
+	case '/':
+		replaceAll := sw.scanner.Peek() == '/'
+		if replaceAll {
+			sw.scanner.Next()
+		}
+
+		pattern, _, err := sw.processStopOn('/', true)
+		if err != nil {
+			if sw.scanner.Peek() == scanner.EOF {
+				return "", errors.New("syntax error: missing '/' in ${}")
+			}
+			return "", err
+		}
+
+		replacement, _, err := sw.processStopOn('}', true)
+		if err != nil {
+			if sw.scanner.Peek() == scanner.EOF {
+				return "", errors.New("syntax error: missing '}'")
+			}
+			return "", err
+		}
+
+		value, set := sw.getEnv(name)
+		if sw.skipUnsetEnv && !set {
+			return fmt.Sprintf("${%s/%s/%s}", name, pattern, replacement), nil
+		}
+
+		re, err := convertShellPatternToRegex(pattern, true, false)
+		if err != nil {
+			return "", errors.Errorf("invalid pattern (%s) in substitution: %s", pattern, err)
+		}
+		if replaceAll {
+			value = re.ReplaceAllString(value, replacement)
+		} else {
+			if idx := re.FindStringIndex(value); idx != nil {
+				value = value[0:idx[0]] + replacement + value[idx[1]:]
+			}
+		}
+		return value, nil
 	default:
 		return "", errors.Errorf("unsupported modifier (%s) in substitution", chs)
 	}
@@ -502,14 +541,16 @@ func BuildEnvs(env []string) map[string]string {
 // Based on
 // https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_13
 // but without the bracket expressions (`[]`)
-func convertShellPatternToRegex(pattern string, greedy bool) (*regexp.Regexp, error) {
+func convertShellPatternToRegex(pattern string, greedy bool, anchored bool) (*regexp.Regexp, error) {
 	var s scanner.Scanner
 	s.Init(strings.NewReader(pattern))
 	var out strings.Builder
 	out.Grow(len(pattern) + 4)
 
 	// match only at the beginning of the string
-	out.WriteByte('^')
+	if anchored {
+		out.WriteByte('^')
+	}
 
 	// default: non-greedy wildcards
 	starPattern := ".*?"
@@ -526,9 +567,9 @@ func convertShellPatternToRegex(pattern string, greedy bool) (*regexp.Regexp, er
 			out.WriteByte('.')
 			continue
 		case '\\':
-			// } as part of ${} needs to be escaped, but the escape isn't part
+			// } and / as part of ${} need to be escaped, but the escape isn't part
 			// of the pattern
-			if s.Peek() == '}' {
+			if s.Peek() == '}' || s.Peek() == '/' {
 				continue
 			}
 			out.WriteRune('\\')
@@ -547,7 +588,7 @@ func convertShellPatternToRegex(pattern string, greedy bool) (*regexp.Regexp, er
 }
 
 func trimPrefix(word, value string, greedy bool) (string, error) {
-	re, err := convertShellPatternToRegex(word, greedy)
+	re, err := convertShellPatternToRegex(word, greedy, true)
 	if err != nil {
 		return "", errors.Errorf("invalid pattern (%s) in substitution: %s", word, err)
 	}

frontend/dockerfile/shell/lex_test.go

@@ -18,15 +18,15 @@ func TestConvertShellPatternToRegex(t *testing.T) {
 		"(()[]{\\}^$.\\*\\?|\\\\": "^\\(\\(\\)\\[\\]\\{\\}\\^\\$\\.\\*\\?\\|\\\\",
 	}
 	for pattern, expected := range cases {
-		res, err := convertShellPatternToRegex(pattern, true)
+		res, err := convertShellPatternToRegex(pattern, true, true)
 		require.NoError(t, err)
 		require.Equal(t, expected, res.String())
 	}
 	invalid := []string{
 		"\\", "\\x", "\\\\\\",
 	}
 	for _, pattern := range invalid {
-		_, err := convertShellPatternToRegex(pattern, true)
+		_, err := convertShellPatternToRegex(pattern, true, true)
 		require.Error(t, err)
 	}
 }
@@ -442,6 +442,39 @@ func TestProcessWithMatches(t *testing.T) {
 			expected: "a",
 			matches:  map[string]struct{}{"FOO": {}},
 		},
+		{
+			// test: wildcards
+			input:    "${FOO/$NEEDLE/.} - ${FOO//$NEEDLE/.}",
+			envs:     map[string]string{"FOO": "/foo*/*/*.txt", "NEEDLE": "\\*/"},
+			expected: "/foo.*/*.txt - /foo..*.txt",
+			matches:  map[string]struct{}{"FOO": {}, "NEEDLE": {}},
+		},
+		{
+			// test: / in patterns
+			input:    "${FOO/$NEEDLE/} - ${FOO//$NEEDLE/}",
+			envs:     map[string]string{"FOO": "/tmp/tmp/bar.txt", "NEEDLE": "/tmp"},
+			expected: "/tmp/bar.txt - /bar.txt",
+			matches:  map[string]struct{}{"FOO": {}, "NEEDLE": {}},
+		},
+		{
+			input:    "${FOO/$NEEDLE/$REPLACEMENT} - ${FOO//$NEEDLE/$REPLACEMENT}",
+			envs:     map[string]string{"FOO": "/a/foo/b/c.txt", "NEEDLE": "/?/", "REPLACEMENT": "/"},
+			expected: "/foo/b/c.txt - /foo/c.txt",
+			matches:  map[string]struct{}{"FOO": {}, "NEEDLE": {}, "REPLACEMENT": {}},
+		},
+		{
+			input:    "${FOO/$NEEDLE/$REPLACEMENT}",
+			envs:     map[string]string{"FOO": "http://google.de", "NEEDLE": "http://", "REPLACEMENT": "https://"},
+			expected: "https://google.de",
+			matches:  map[string]struct{}{"FOO": {}, "NEEDLE": {}, "REPLACEMENT": {}},
+		},
+		{
+			// test: substitute escaped separator characters
+			input:    "${FOO//\\//\\/}",
+			envs:     map[string]string{"FOO": "/tmp/foo.txt"},
+			expected: "\\/tmp\\/foo.txt",
+			matches:  map[string]struct{}{"FOO": {}},
+		},
 	}
 
 	for _, c := range tc {