diff --git a/README.md b/README.md index 6f4d5b5..ef2fa5c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,31 @@ -# gluaspider -spider tool +## go-nsq + +[![GoDoc](https://godoc.org/github.com/kfrico/gluaspider?status.svg)](https://godoc.org/github.com/kfrico/gluaspider) [![GitHub release](https://img.shields.io/github/release/kfrico/gluaspider.svg)](https://github.com/kfrico/gluaspider/releases/latest) + +simple spider module for [gopher-lua](https://github.com/yuin/gopher-lua) + +### Docs + +See [godoc](https://godoc.org/github.com/kfrico/gluaspider) + +### Installation + +``` +go get github.com/kfrico/gluaspider +``` + + +### Usage + +``` +import ( + "github.com/kfrico/gluaspider" +) + +L := lua.NewState() +defer L.Close() + +L.PreloadModule("spider", gluaspider.NewSpider().Loader) +``` + +### Tests \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4b1559d --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module github.com/kfrico/gluaspider + +go 1.12 + +require ( + github.com/PuerkitoBio/goquery v1.5.1 + github.com/go-resty/resty/v2 v2.2.0 + github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb + golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect + golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect + golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect + golang.org/x/text v0.3.2 // indirect + golang.org/x/tools v0.0.0-20200305224536-de023d59a5d1 // indirect + layeh.com/gopher-luar v1.0.7 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..76e49b5 --- /dev/null +++ b/go.sum @@ -0,0 +1,41 @@ +github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= +github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= +github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= +github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/go-resty/resty/v2 v2.2.0 h1:vgZ1cdblp8Aw4jZj3ZsKh6yKAlMg3CHMrqFSFFd+jgY= +github.com/go-resty/resty/v2 v2.2.0/go.mod h1:nYW/8rxqQCmI3bPz9Fsmjbr2FBjGuR2Mzt6kDh3zZ7w= +github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb h1:ZkM6LRnq40pR1Ox0hTHlnpkcOTuFIDQpZ1IN8rKKhX0= +github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200305224536-de023d59a5d1/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +layeh.com/gopher-luar v1.0.7 h1:53iv6CCkRs5wyofZ+qVXcyAYQOIG52s6pt4xkqZdq7k= +layeh.com/gopher-luar v1.0.7/go.mod h1:TPnIVCZ2RJBndm7ohXyaqfhzjlZ+OA2SZR/YwL8tECk= diff --git a/spider.go b/spider.go new file mode 100644 index 0000000..6d99e3c --- /dev/null +++ b/spider.go @@ -0,0 +1,80 @@ +package gluaspider + +import ( + "strings" + + "github.com/PuerkitoBio/goquery" + "github.com/go-resty/resty/v2" + "github.com/yuin/gopher-lua" + "layeh.com/gopher-luar" +) + +// spider type +type spider struct { + restyClient *resty.Client +} + +// NewSpider NewSpider +func NewSpider() *spider { + return &spider{ + restyClient: resty.New(), + } +} + +// Get Simple Get Url +func (s *spider) Get(l *lua.LState) int { + resp, err := s.restyClient.R().Get(l.CheckString(1)) + + if err != nil { + l.Push(lua.LNil) + l.Push(lua.LString(err.Error())) + + return 2 + } + + return s.newDocumentFromString(l, resp.String()) +} + +// RestyClient Get RestyClient +func (s *spider) RestyClient(l *lua.LState) int { + l.Push(luar.New(l, s.restyClient)) + + return 1 +} + +// newDocumentFromString New Goquery Document From String +func (s *spider) newDocumentFromString(l *lua.LState, html string) int { + doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) + + if err != nil { + l.Push(lua.LNil) + l.Push(lua.LString(err.Error())) + + return 2 + } + + l.Push(luar.New(l, doc)) + l.Push(lua.LNil) + + return 2 +} + +// NewDocumentFromString New Goquery Document From String +func (s *spider) NewDocumentFromString(l *lua.LState) int { + return s.newDocumentFromString(l, l.CheckString(1)) +} + +// Loader Loader +func (s *spider) Loader(l *lua.LState) int { + // register functions to the table + mod := l.SetFuncs(l.NewTable(), map[string]lua.LGFunction{ + "RestyClient": s.RestyClient, + "NewDocumentFromString": s.NewDocumentFromString, + "Get": s.Get, + }) + + // returns the module + l.Push(mod) + + return 1 +}