golang: support all: prefix for go:embed directives (Cherry-pick of

#21559) (#21564)

Support the `all:` prefix for the `go:embed` directive. This directive includes files which start with `_` or `.` which are ordinarily excluded.

See Go commit golang/go@36dbf7f which added this support for the details.

Closes #21554

---------

Co-authored-by: Tom Dyas <tom@shoalsoftware.com>

docs/notes/2.22.x.md

@@ -141,6 +141,8 @@ Setting [the `orphan_files_behaviour = "ignore"` option](https://www.pantsbuild.
 
 Fix a bug where Pants raised an internal exception which occurred when compiling a Go package with coverage support when the package also had an external test which imported the base package.
 
+Add support for the `all:` prefix to patterns used with the `go:embed` directive. The `all:` prefix includes files which start with `_` or `.` which are ordinarilly excluded.
+
 ### Plugin API changes
 
 The `PythonToolRequirementsBase` and `PythonToolBase` classes now have a new `help_short` field. Subclasses should now use `help_short` instead of the `help` field. The `help` field will be automatically generated using `help_short`, and will include the tool's default package version and provide instructions on how to override this version using a custom lockfile.

src/python/pants/backend/go/go_sources/embedcfg/main.go

@@ -215,6 +215,12 @@ func resolveEmbed(pattern string, root *embedNode) (matchedPaths, matchedFiles [
 		}
 	}()
 
+	excludeHiddenNames := true
+	if strings.HasPrefix(pattern, "all:") {
+		pattern = pattern[len("all:"):]
+		excludeHiddenNames = false
+	}
+
 	// Check that the pattern has valid syntax.
 	if _, err := path.Match(pattern, ""); err != nil || !validEmbedPattern(pattern) {
 		return nil, nil, fmt.Errorf("invalid pattern syntax")
@@ -238,8 +244,10 @@ func resolveEmbed(pattern string, root *embedNode) (matchedPaths, matchedFiles [
 		// See golang/go#42328.
 		matchTreeErr := matchNode.walk(func(childRel string, childNode *embedNode) error {
 			if childRel != "" {
-				if base := path.Base(childRel); strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") {
-					return errSkip
+				if excludeHiddenNames {
+					if base := path.Base(childRel); strings.HasPrefix(base, ".") || strings.HasPrefix(base, "_") {
+						return errSkip
+					}
 				}
 			}
 			if !childNode.isDir() {

src/python/pants/backend/go/util_rules/embed_integration_test.py

@@ -27,7 +27,7 @@
 from pants.backend.go.util_rules.embedcfg import EmbedConfig
 from pants.build_graph.address import Address
 from pants.core.goals.test import TestResult, get_filtered_environment
-from pants.core.target_types import ResourceTarget
+from pants.core.target_types import ResourcesGeneratorTarget, ResourceTarget
 from pants.core.util_rules import source_files
 from pants.testutil.rule_runner import QueryRule, RuleRunner
 
@@ -55,6 +55,7 @@ def rule_runner() -> RuleRunner:
             GoModTarget,
             GoPackageTarget,
             ResourceTarget,
+            ResourcesGeneratorTarget,
         ],
     )
     rule_runner.set_options(["--go-test-args=-v -bench=."], env_inherit={"PATH"})
@@ -341,3 +342,94 @@ def test_third_party_package_embed(rule_runner: RuleRunner) -> None:
         TestResult, [GoTestRequest.Batch("", (GoTestFieldSet.create(tgt),), None)]
     )
     _assert_test_result_success(result)
+
+
+def test_embed_with_all_prefix(rule_runner: RuleRunner) -> None:
+    rule_runner.write_files(
+        {
+            "BUILD": dedent(
+                """
+                go_mod(name='mod')
+                go_package(name='pkg', dependencies=[":resources"])
+                resources(name='resources', sources=['r/**'])
+                """
+            ),
+            "go.mod": dedent(
+                """\
+                module go.example.com/foo
+                go 1.17
+                """
+            ),
+            "r/_foo/hello.txt": "hello",
+            "r/_foo/.another-config.txt": "world",
+            "r/xyzzy.txt": "another",
+            "r/.config.txt": "some-config-file",
+            "foo.go": dedent(
+                """\
+                package foo
+                """
+            ),
+            "foo_test.go": dedent(
+                """\
+                package foo
+                import (
+                    "embed"
+                    "errors"
+                    "os"
+                    "testing"
+                )
+
+                //go:embed all:r
+                var allResources embed.FS
+
+                //go:embed r
+                var resources embed.FS
+
+                func assertFileContent(t *testing.T, fsys embed.FS, name string, content string) {
+                    t.Helper()
+
+                    rawContent, err := fsys.ReadFile(name)
+                    if err != nil {
+                        t.Error(err)
+                        return
+                    }
+
+                    if string(rawContent) != content {
+                        t.Errorf("read %v = %q, want %q", name, rawContent, content)
+                    }
+                }
+
+                func assertFileDoesNotExist(t *testing.T, fsys embed.FS, name string) {
+                    _, err := fsys.ReadFile(name)
+                    if err == nil {
+                        t.Errorf("expected file %v to not exist but it actually exists", name)
+                        return
+                    }
+
+                    if !errors.Is(err, os.ErrNotExist) {
+                        t.Error(err)
+                    }
+                }
+
+                func TestAllResources(t *testing.T) {
+                    assertFileContent(t, allResources, "r/_foo/hello.txt", "hello")
+                    assertFileContent(t, allResources, "r/_foo/.another-config.txt", "world")
+                    assertFileContent(t, allResources, "r/xyzzy.txt", "another")
+                    assertFileContent(t, allResources, "r/.config.txt", "some-config-file")
+                }
+
+                func TestResources(t *testing.T) {
+                    assertFileDoesNotExist(t, resources, "r/_foo/hello.txt")
+                    assertFileDoesNotExist(t, resources, "r/_foo/.another-config.txt")
+                    assertFileContent(t, resources, "r/xyzzy.txt", "another")
+                    assertFileDoesNotExist(t, resources, "r/.config.txt")
+                }
+                """
+            ),
+        }
+    )
+    tgt = rule_runner.get_target(Address("", target_name="pkg"))
+    result = rule_runner.request(
+        TestResult, [GoTestRequest.Batch("", (GoTestFieldSet.create(tgt),), None)]
+    )
+    _assert_test_result_success(result)