chore: refactor parser + testing + ci (#31)

* wip: parser test

* unit test workflow

* ignore configs from embed

* cleanup RemoveTagged lines and Cmt Swap

* refactor `RemoveModuleFromText` + test

* cleanup

* minor

.github/workflows/unit-test.yaml

@@ -0,0 +1,37 @@
+---
+    name: unit-test
+
+    on:
+      push:
+
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+
+    env:
+      GO_VERSION: 1.21.0
+
+    jobs:
+      build:
+        runs-on: ubuntu-latest
+        name: build
+        steps:
+          - uses: actions/checkout@v4
+          - name: Setup go
+            uses: actions/setup-go@v4
+            with:
+              go-version: ${{ env.GO_VERSION }}
+          - run: make install
+
+      test:
+        runs-on: ubuntu-latest
+        name: test
+        steps:
+          - name: Install Go
+            uses: actions/setup-go@v4
+            with:
+              go-version: ${{ env.GO_VERSION }}
+          - name: Checkout code
+            uses: actions/checkout@v4
+          - name: Test
+            run: go test ./...

cmd/spawn/new-chain.go

@@ -11,7 +11,7 @@ import (
 )
 
 var (
-	SupportedModules = items{
+	SupportedFeatures = items{
 		{ID: "tokenfactory", IsSelected: true, Details: "Native token minting, sending, and burning on the chain"},
 		{ID: "poa", IsSelected: true, Details: "Proof-of-Authority consensus algorithm (permissioned network)"},
 		{ID: "globalfee", IsSelected: true, Details: "Static minimum fee(s) for all transactions, controlled by governance"},
@@ -35,7 +35,7 @@ func init() {
 	newChain.Flags().StringP(FlagBinDaemon, "b", "simd", "binary name")
 	newChain.Flags().String(FlagGithubOrg, "rollchains", "github organization")
 	newChain.Flags().String(FlagTokenDenom, "token", "bank token denomination")
-	newChain.Flags().StringSlice(FlagDisabled, []string{}, "disable features: "+SupportedModules.String())
+	newChain.Flags().StringSlice(FlagDisabled, []string{}, "disable features: "+SupportedFeatures.String())
 	newChain.Flags().Bool(FlagDebugging, false, "enable debugging")
 	newChain.Flags().Bool(FlagNoGit, false, "git init base")
 	newChain.Flags().Bool(FlagBypassPrompt, false, "bypass UI prompt")
@@ -66,7 +66,7 @@ var newChain = &cobra.Command{
 
 		bypassPrompt, _ := cmd.Flags().GetBool(FlagBypassPrompt)
 		if len(disabled) == 0 && !bypassPrompt {
-			items, err := selectItems(0, SupportedModules, true)
+			items, err := selectItems(0, SupportedFeatures, true)
 			if err != nil {
 				fmt.Println("Error selecting disabled:", err)
 				return

cmd/spawn/ui.go

@@ -58,7 +58,7 @@ func selectItems(selectedPos int, allItems items, returnOpposite bool) (items, e
 	}
 
 	prompt := promptui.Select{
-		Label:        "Module Selector",
+		Label:        "Feature Selector",
 		Items:        allItems,
 		Templates:    templates,
 		Size:         10,

go.mod

@@ -10,10 +10,14 @@ require (
 	github.com/spf13/cobra v1.8.0
 	github.com/spf13/pflag v1.0.5
 	github.com/strangelove-ventures/simapp v0.0.0-00000000-000000000000
+	github.com/stretchr/testify v1.8.4
 )
 
 require (
 	github.com/chzyer/readline v1.5.1 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	golang.org/x/sys v0.17.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -10,18 +10,26 @@ github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38
 github.com/cosmos/btcutil v1.0.5 h1:t+ZFcX77LpKtDBhjucvnOH8C2l2ioGsBNEQ3jef8xFk=
 github.com/cosmos/btcutil v1.0.5/go.mod h1:IyB7iuqZMJlthe2tkIFL33xPyzbFYP0XVdS8P5lUPis=
 github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
 github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA=
 github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
 github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

simapp/app/app.go

@@ -679,11 +679,11 @@ func NewChainApp(
 
 	wasmDir := filepath.Join(homePath, "wasm")
 	wasmConfig, err := wasm.ReadWasmConfig(appOpts)
-	// !spawntag:wasm
+	// <spawntag:wasm
 	if err != nil {
 		panic(fmt.Sprintf("error while reading wasm config: %s", err))
 	}
-	// !spawntag:wasm
+	// spawntag:wasm>
 
 	// The last arguments can contain custom message handlers, and custom query handlers,
 	// if we want to allow any custom callbacks
@@ -1014,7 +1014,7 @@ func NewChainApp(
 	return app
 }
 
-// !spawntag:globalfee
+// <spawntag:globalfee
 func GetDefaultBypassFeeMessages() []string {
 	return []string{
 		sdk.MsgTypeURL(&ibcchanneltypes.MsgRecvPacket{}),
@@ -1031,7 +1031,7 @@ func GetDefaultBypassFeeMessages() []string {
 	}
 }
 
-//!spawntag:globalfee
+// spawntag:globalfee>
 
 func (app *ChainApp) FinalizeBlock(req *abci.RequestFinalizeBlock) (*abci.ResponseFinalizeBlock, error) {
 	// when skipping sdk 47 for sdk 50, the upgrade handler is called too late in BaseApp

simapp/embed.go

@@ -8,7 +8,7 @@ import (
 
 // TODO: x/ and proto/ in the future
 //
-//go:embed .github/* app/* chains/* cmd/* configs/* contrib/* scripts/* Makefile Dockerfile *.*
+//go:embed .github/* app/* chains/* cmd/* contrib/* scripts/* Makefile Dockerfile *.*
 var SimAppFS embed.FS
 
 //go:embed interchaintest/*

spawn/parser.go

@@ -6,130 +6,155 @@ import (
 )
 
 const (
-	expectedFormat = "// spawntag:"
-	commentFormat  = "?spawntag:"
+	// StdFormat is the standard format for removing a line if a feature is removed.
+	StdFormat = "spawntag:%s"
+
+	// ExpectedFormat is the standard format for removing a line if a module is removed.
+	// e.g. // spawntag:tokenfactory would remove the line if tokenfactory is removed.
+	// NOTE: This is not user facing, and is only used for internal parsing of the simapp.
+	ExpectedFormat = "// spawntag:"
+
+	// CommentSwapFormat is the format for swapping a line with another if a module is removed.
+	CommentSwapFormat = "?spawntag:%s"
+
+	// MultiLineStartFormat is the format for starting a multi-line comment which removes all text
+	// until the end of the comment.
+	// <spawntag:[searchTerm]
+	MultiLineStartFormat = "<" + StdFormat
+
+	// spawntag:[searchTerm]>
+	MultiLineEndFormat = StdFormat + ">"
 )
 
 // Sometimes we remove a module line and would like to swap it for another.
 func (fc *FileContent) HandleCommentSwaps(name string) {
-	newContent := make([]string, 0, len(strings.Split(fc.Contents, "\n")))
-
-	uncomment := fmt.Sprintf("%s:%s", commentFormat, name)
+	splitContent := strings.Split(fc.Contents, "\n")
+	tag := fmt.Sprintf(CommentSwapFormat, name)
 
-	for idx, line := range strings.Split(fc.Contents, "\n") {
-		hasUncommentTag := strings.Contains(line, uncomment)
-		if hasUncommentTag {
-			line = strings.Replace(line, "//", "", 1)
-			line = strings.TrimRight(strings.Replace(line, fmt.Sprintf("// %s", uncomment), "", 1), " ")
-			fmt.Printf("uncomment %s: %d, %s\n", name, idx, line)
+	for idx, line := range splitContent {
+		// If the line does not have the comment swap tag, then continue
+		if !strings.Contains(line, tag) {
+			continue
 		}
 
-		newContent = append(newContent, line)
+		// removes the // spawntag:[name] comment from the end of the source code
+		line = removeSpawnTagLineComment(line, tag)
+
+		// uncomments the line (to expose the source code for application usage)
+		line = uncommentLineSource(line)
+
+		// Since we are just uncommenting the line, it's safe to just replace the line at the index
+		splitContent[idx] = line
+
 	}
 
-	fc.Contents = strings.Join(newContent, "\n")
+	fc.Contents = strings.Join(splitContent, "\n")
 }
 
 // RemoveTaggedLines deletes tagged lines or just removes the comment if desired.
 func (fc *FileContent) RemoveTaggedLines(name string, deleteLine bool) {
-	newContent := make([]string, 0, len(strings.Split(fc.Contents, "\n")))
+	splitContent := strings.Split(fc.Contents, "\n")
+	newContent := make([]string, 0, len(splitContent))
 
-	startIdx := -1
-	for idx, line := range strings.Split(fc.Contents, "\n") {
-		hasTag := strings.Contains(line, fmt.Sprintf("spawntag:%s", name))
-		hasMultiLineTag := strings.Contains(line, fmt.Sprintf("!spawntag:%s", name))
+	startMultiLineDelete := false
+	for idx, line := range splitContent {
 
 		// if the line has a tag, and the tag starts with a !, then we will continue until we
 		// find the end of the tag with another.
-		if startIdx != -1 {
-			if !hasMultiLineTag {
+		if startMultiLineDelete {
+			hasMultiLineEndTag := strings.Contains(line, fmt.Sprintf(MultiLineEndFormat, name))
+			if !hasMultiLineEndTag {
 				continue
 			}
 
-			startIdx = -1
+			// the line which has the closing multiline end tag, we then continue to add lines as normal
+			startMultiLineDelete = false
 			fmt.Println("endIdx:", idx, line)
 			continue
 		}
 
-		if hasMultiLineTag {
+		// <spawntag:[searchTerm]
+		if strings.Contains(line, fmt.Sprintf(MultiLineStartFormat, name)) {
 			if !deleteLine {
 				continue
 			}
 
-			startIdx = idx
+			startMultiLineDelete = true
 			fmt.Printf("startIdx %s: %d, %s\n", name, idx, line)
 			continue
 		}
 
-		if hasTag {
+		// remove a line if it contains spawntag:[searchTerm]
+		if strings.Contains(line, fmt.Sprintf(StdFormat, name)) {
 			if deleteLine {
 				continue
 			}
 
-			line = removeJustSpawnTagLineComment(line)
+			line = removeSpawnTagLineComment(line, ExpectedFormat)
 		}
 
 		newContent = append(newContent, line)
 	}
 
-	// return []byte(strings.Join(newContent, "\n"))
 	fc.Contents = strings.Join(newContent, "\n")
 }
 
-// removeLineComment removes just the spawntag comment from a line of code.
-// this way it is not user facing
-func removeJustSpawnTagLineComment(line string) string {
+// removeSpawnTagLineComment removes just the spawntag comment from a line of code.
+func removeSpawnTagLineComment(line string, tag string) string {
 	// QOL for us to not tear our hair out if we have a space or not
 	// Could do this for all contents on load?
-	line = strings.ReplaceAll(line, "//spawntag:", expectedFormat)
+	line = strings.ReplaceAll(line, "//spawntag:", ExpectedFormat)
 
-	line = strings.Split(line, expectedFormat)[0]
+	line = strings.Split(line, fmt.Sprintf("// %s", tag))[0]
 	return strings.TrimRight(line, " ")
 }
 
 // RemoveGeneralModule removes any matching names from the fileContent.
 // i.e. if moduleFind is "tokenfactory" any lines with "tokenfactory" will be removed
 // including comments.
 // If an import or other line depends on a solo module a user wishes to remove, add a comment to the line
-// such as `// tag:tokenfactory` to also remove other lines within the simapp template
+// such as `// spawntag:tokenfactory` to also remove other lines within the simapp template
 func (fc *FileContent) RemoveModuleFromText(removeText string, pathSuffix ...string) {
 	if !fc.InPaths(pathSuffix) {
 		return
 	}
 
-	newContent := make([]string, 0, len(strings.Split(fc.Contents, "\n")))
+	splitContent := strings.Split(fc.Contents, "\n")
+	newContent := make([]string, 0, len(splitContent))
 
-	startIdx := -1
-	for idx, line := range strings.Split(fc.Contents, "\n") {
-		// if we are in a startIdx, then we need to continue until we find the close parenthesis (i.e. NewKeeper)
-		if startIdx != -1 {
+	startBatchDelete := false
+	for idx, line := range splitContent {
+		// if line contains //spawntag:ignore then we use that line.
+		// useful if some text is 'wasm' as a bech32 prefix, not a variable / type we need to remove.
+		if strings.Contains(line, fmt.Sprintf(StdFormat, "ignore")) {
+			fmt.Printf("Ignoring removal: %s: %d, %s\n", removeText, idx, line)
+			newContent = append(newContent, line)
+			continue
+		}
+
+		// if we are in a batch delete, then we need to continue until we find the close parenthesis or bracket
+		// (i.e. NewKeeper in app.go is a good example fo this)
+		if startBatchDelete {
 			fmt.Printf("rm %s startIdx: %d, %s\n", removeText, idx, line)
+
 			if strings.TrimSpace(line) == ")" || strings.TrimSpace(line) == "}" {
 				fmt.Println("endIdx:", idx, line)
-				startIdx = -1
+				startBatchDelete = false
 				continue
 			}
 
 			continue
 		}
 
-		lineHas := strings.Contains(line, removeText)
-
-		// if line contains //ignore or // ignore, then we use that line
-		// useful if some text is 'wasm' as a bech32 prefix, not a variable / type.
-		if hasIgnoreComment(line) {
-			fmt.Printf("Ignoring removal: %s: %d, %s\n", removeText, idx, line)
-			newContent = append(newContent, line)
-			continue
-		}
-
-		if lineHas && (strings.HasSuffix(strings.TrimSpace(line), "(") || strings.HasSuffix(strings.TrimSpace(line), "{")) {
-			startIdx = idx
-			fmt.Printf("startIdx %s: %d, %s\n", removeText, idx, line)
-			continue
-		}
+		// if the line has the text we wish to remove, begin the removal process.
+		if strings.Contains(line, removeText) {
+			// if the line ends with an opening symbol, we start a batch delete process
+			if DoesLineEndWithOpenSymbol(line) {
+				startBatchDelete = true
+				fmt.Printf("startIdx %s: %d, %s\n", removeText, idx, line)
+				continue
+			}
 
-		if lineHas {
 			fmt.Printf("rm %s: %d, %s\n", removeText, idx, line)
 			continue
 		}
@@ -140,6 +165,26 @@ func (fc *FileContent) RemoveModuleFromText(removeText string, pathSuffix ...str
 	fc.Contents = strings.Join(newContent, "\n")
 }
 
-func hasIgnoreComment(line string) bool {
-	return strings.Contains(line, "//ignore") || strings.Contains(line, "// ignore")
+// doesLineEndWithOpenSymbol returns true if the end of a line opens a statement such as a multi-line function.
+func DoesLineEndWithOpenSymbol(line string) bool {
+	// remove comment if there is one
+	if strings.Contains(line, "//") {
+		line = strings.Split(line, "//")[0]
+	}
+
+	return strings.HasSuffix(strings.TrimSpace(line), "(") || strings.HasSuffix(strings.TrimSpace(line), "{")
+}
+
+// getCommentText returns the trimmed text from a line comment.
+func getCommentText(line string) string {
+	if strings.Contains(line, "//") {
+		text := strings.Split(line, "//")[1]
+		return strings.TrimSpace(text)
+	}
+
+	return ""
+}
+
+func uncommentLineSource(line string) string {
+	return strings.Replace(line, "//", "", 1)
 }