From 3efd973e580b606993c325acc984560ca423d75e Mon Sep 17 00:00:00 2001 From: at15 Date: Sun, 9 Jul 2017 01:32:17 -0700 Subject: [PATCH] [play] Try built in template to replace pongo2 - for #2, we may not need that much feature on pongo2 - the main drawback of golang's built in template is its syntax is quite strange - golang's template does support remove new line, though it would also trim tab --- config/README.md | 2 + config/pkg.go | 7 ++- config/pong2_test.go | 1 - config/pongo2.go | 2 + config/yaml.go | 14 +++-- playground/README.md | 4 ++ playground/std/std_template_test.go | 83 +++++++++++++++++++++++++++++ requests/doc.go | 2 +- requests/requests.go | 2 +- requests/requests_test.go | 6 +-- util/env.go | 1 + 11 files changed, 109 insertions(+), 15 deletions(-) create mode 100644 playground/README.md create mode 100644 playground/std/std_template_test.go diff --git a/config/README.md b/config/README.md index 6a0af55..f4611c6 100644 --- a/config/README.md +++ b/config/README.md @@ -28,6 +28,8 @@ though for most formats, go already have encoder and decoder Without using multiple document +- [ ] TODO: there is not function as `env()` shown in the example + ````yaml vars: influxdb_port: 8080 diff --git a/config/pkg.go b/config/pkg.go index 4bd3e36..b2a6834 100644 --- a/config/pkg.go +++ b/config/pkg.go @@ -6,4 +6,9 @@ import ( var log = util.Logger.NewEntryWithPkg("gommon.config") -var defaultKeyDelimiter = "." +const ( + yamlDocumentSeparator = "---" + pongo2DefaultBaseDir = "" + pongo2DefaultSetName = "gommon-yaml" + defaultKeyDelimiter = "." +) diff --git a/config/pong2_test.go b/config/pong2_test.go index 92ad00e..1637c48 100644 --- a/config/pong2_test.go +++ b/config/pong2_test.go @@ -14,5 +14,4 @@ func TestRenderDocument(t *testing.T) { //t.Log(err) assert.Nil(err) assert.Equal("bar and 1 and 1", out) - } diff --git a/config/pongo2.go b/config/pongo2.go index eb77286..d65bc18 100644 --- a/config/pongo2.go +++ b/config/pongo2.go @@ -12,6 +12,7 @@ var ( ) // RenderDocumentString uses defaultSet due to pongo2's strange API +// Deprecated use the methods in YAMLConfig instead func RenderDocumentString(tplStr string, context pongo2.Context) (string, error) { // pongo2.Context{} is just map[string]interface{} // FIXME: pongo2.FromString is not longer in the new API, must first create a set @@ -27,6 +28,7 @@ func RenderDocumentString(tplStr string, context pongo2.Context) (string, error) } // RenderDocumentBytes uses defaultSet and since pongo2 have two function for String and Bytes, the wrapper also has two function +// Deprecated use the methods in YAMLConfig instead func RenderDocumentBytes(tplBytes []byte, context pongo2.Context) ([]byte, error) { tpl, err := defaultSet.FromBytes(tplBytes) var out []byte diff --git a/config/yaml.go b/config/yaml.go index 6bffe9b..fb9e57e 100644 --- a/config/yaml.go +++ b/config/yaml.go @@ -15,14 +15,12 @@ import ( "gopkg.in/yaml.v2" ) -const documentSeparator = "---" - // YAMLConfig is a thread safe struct for parse YAML file and get value type YAMLConfig struct { vars map[string]interface{} data map[string]interface{} keyDelimiter string - mu sync.RWMutex // TODO: may use RWMutex + mu sync.RWMutex loader pongo2.TemplateLoader set *pongo2.TemplateSet } @@ -30,7 +28,7 @@ type YAMLConfig struct { // SplitMultiDocument splits a yaml file that contains multiple documents and // (only) trim the first one if it is empty func SplitMultiDocument(data []byte) [][]byte { - docs := bytes.Split(data, []byte(documentSeparator)) + docs := bytes.Split(data, []byte(yamlDocumentSeparator)) // check the first one, it could be empty if len(docs[0]) != 0 { return docs @@ -43,8 +41,8 @@ func NewYAMLConfig() *YAMLConfig { c := new(YAMLConfig) c.clear() c.keyDelimiter = defaultKeyDelimiter - c.loader = pongo2.MustNewLocalFileSystemLoader("") - c.set = pongo2.NewSet("gommon-yaml", c.loader) + c.loader = pongo2.MustNewLocalFileSystemLoader(pongo2DefaultBaseDir) + c.set = pongo2.NewSet(pongo2DefaultSetName, c.loader) return c } @@ -71,12 +69,12 @@ func (c *YAMLConfig) ParseSingleDocumentBytes(doc []byte) error { c.mu.Lock() defer c.mu.Unlock() + // we render the template twice, first time we use vars from previous documents and environment variables + // second time, we use vars declared in this document, if any. pongoContext := pongo2.Context{ "vars": c.vars, "envs": util.EnvAsMap(), } - // we render the template twice, first time we use vars from previous documents and environment variables - // second time, we use vars declared in this document, if any. // this is the first render rendered, err := c.RenderDocumentBytes(doc, pongoContext) if err != nil { diff --git a/playground/README.md b/playground/README.md new file mode 100644 index 0000000..c597ccb --- /dev/null +++ b/playground/README.md @@ -0,0 +1,4 @@ +# Playground + +For testing language semantics and small benchmarks. Prototype and library examples are also put here. +It also keeps some minimal code for reproduce/solve issues. \ No newline at end of file diff --git a/playground/std/std_template_test.go b/playground/std/std_template_test.go new file mode 100644 index 0000000..7bcdb6d --- /dev/null +++ b/playground/std/std_template_test.go @@ -0,0 +1,83 @@ +package std + +import ( + "os" + "text/template" + "testing" + "bytes" + + asst "github.com/stretchr/testify/assert" +) + +func TestTemplate_Funcs(t *testing.T) { + assert := asst.New(t) + const tmplText = ` + {{env "HOME"}} + {{var "foo"}} +` + vars := map[string]string{"foo": "bar"} + + funcMap := template.FuncMap{ + "env": func(name string) string { + return os.Getenv(name) + }, + "var": func(name string) string { + return vars[name] + }, + } + + tmpl, err := template.New("funcs test").Funcs(funcMap).Parse(tmplText) + if err != nil { + t.Fatal(err) + } + + var b bytes.Buffer + assert.Nil(tmpl.Execute(&b, "")) + rendered := b.String() + assert.Contains(rendered, os.Getenv("HOME"), "env function in template should be called") + assert.Contains(rendered, "bar") + + // now we update vars + vars["foo"] = "bar2" + b.Reset() + assert.Nil(tmpl.Execute(&b, "")) + rendered = b.String() + assert.Contains(rendered, "bar2", "var function should be using the updated value") +} + + +func TestTemplate_Range(t *testing.T) { + assert := asst.New(t) + // NOTE: the `-` is used to remove the following new line https://golang.org/pkg/text/template/#hdr-Text_and_spaces + // FIXME: - will also remove space which would break yaml indent + const tmplText = ` +{{ range $name := var "databases" }} +{{ $name -}}: +{{ $db := var $name }} + name: {{ $name }} + type: {{ $db.type -}} +{{ end }} +` + + vars := map[string]interface{}{"foo": "barr"} + vars["databases"] = []string{"cassandra", "mysql", "xephonk"} + vars["cassandra"] = map[string]string{"type": "nosql"} + vars["mysql"] = map[string]string{"type": "sql"} + vars["xephonk"] = map[string]string{"type": "tsdb"} + + + funcMap := template.FuncMap{ + "env": func(name string) string { + return os.Getenv(name) + }, + "var": func(name string) interface{} { + return vars[name] + }, + } + + tmpl, err := template.New("range test").Funcs(funcMap).Parse(tmplText) + assert.Nil(err) + err = tmpl.Execute(os.Stdout, "") + t.Log(err) + assert.Nil(err) +} \ No newline at end of file diff --git a/requests/doc.go b/requests/doc.go index 9b6646d..beb36e6 100644 --- a/requests/doc.go +++ b/requests/doc.go @@ -1,4 +1,4 @@ /* Package requests is a pythonic HTTP library for Gopher - */ +*/ package requests diff --git a/requests/requests.go b/requests/requests.go index 9ea2719..edc1fef 100644 --- a/requests/requests.go +++ b/requests/requests.go @@ -6,8 +6,8 @@ import ( "io/ioutil" "net/http" - "github.com/pkg/errors" "bytes" + "github.com/pkg/errors" "io" "strings" ) diff --git a/requests/requests_test.go b/requests/requests_test.go index fb46fcb..7851e1d 100644 --- a/requests/requests_test.go +++ b/requests/requests_test.go @@ -1,12 +1,12 @@ package requests import ( - "testing" - "net/http" "fmt" + asst "github.com/stretchr/testify/assert" "io/ioutil" + "net/http" "net/http/httptest" - asst "github.com/stretchr/testify/assert" + "testing" ) func TestRequestsE2E(t *testing.T) { diff --git a/util/env.go b/util/env.go index dceaa1c..833592f 100644 --- a/util/env.go +++ b/util/env.go @@ -6,6 +6,7 @@ import ( ) // EnvAsMap returns environment variables as string map +// TODO: might cache it when package init, the problem of doing so is user might call os.Setenv, we also do this in test func EnvAsMap() map[string]string { //https://coderwall.com/p/kjuyqw/get-environment-variables-as-a-map-in-golang envMap := make(map[string]string)