-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
4 changed files
with
167 additions
and
2 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,2 +1,31 @@ | ||
| # gluaspider | ||
| spider tool | ||
| ## go-nsq | ||
|
|
||
| [](https://godoc.org/github.com/kfrico/gluaspider) [](https://github.com/kfrico/gluaspider/releases/latest) | ||
|
|
||
| simple spider module for [gopher-lua](https://github.com/yuin/gopher-lua) | ||
|
|
||
| ### Docs | ||
|
|
||
| See [godoc](https://godoc.org/github.com/kfrico/gluaspider) | ||
|
|
||
| ### Installation | ||
|
|
||
| ``` | ||
| go get github.com/kfrico/gluaspider | ||
| ``` | ||
|
|
||
|
|
||
| ### Usage | ||
|
|
||
| ``` | ||
| import ( | ||
| "github.com/kfrico/gluaspider" | ||
| ) | ||
| L := lua.NewState() | ||
| defer L.Close() | ||
| L.PreloadModule("spider", gluaspider.NewSpider().Loader) | ||
| ``` | ||
|
|
||
| ### Tests |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,15 @@ | ||
| module github.com/kfrico/gluaspider | ||
|
|
||
| go 1.12 | ||
|
|
||
| require ( | ||
| github.com/PuerkitoBio/goquery v1.5.1 | ||
| github.com/go-resty/resty/v2 v2.2.0 | ||
| github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb | ||
| golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073 // indirect | ||
| golang.org/x/net v0.0.0-20200301022130-244492dfa37a // indirect | ||
| golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527 // indirect | ||
| golang.org/x/text v0.3.2 // indirect | ||
| golang.org/x/tools v0.0.0-20200305224536-de023d59a5d1 // indirect | ||
| layeh.com/gopher-luar v1.0.7 | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,41 @@ | ||
| github.com/PuerkitoBio/goquery v1.5.1 h1:PSPBGne8NIUWw+/7vFBV+kG2J/5MOjbzc7154OaKCSE= | ||
| github.com/PuerkitoBio/goquery v1.5.1/go.mod h1:GsLWisAFVj4WgDibEWF4pvYnkVQBpKBKeU+7zCJoLcc= | ||
| github.com/andybalholm/cascadia v1.1.0 h1:BuuO6sSfQNFRu1LppgbD25Hr2vLYW25JvxHs5zzsLTo= | ||
| github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= | ||
| github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= | ||
| github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= | ||
| github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= | ||
| github.com/go-resty/resty/v2 v2.2.0 h1:vgZ1cdblp8Aw4jZj3ZsKh6yKAlMg3CHMrqFSFFd+jgY= | ||
| github.com/go-resty/resty/v2 v2.2.0/go.mod h1:nYW/8rxqQCmI3bPz9Fsmjbr2FBjGuR2Mzt6kDh3zZ7w= | ||
| github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= | ||
| github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb h1:ZkM6LRnq40pR1Ox0hTHlnpkcOTuFIDQpZ1IN8rKKhX0= | ||
| github.com/yuin/gopher-lua v0.0.0-20191220021717-ab39c6098bdb/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= | ||
| golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= | ||
| golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= | ||
| golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= | ||
| golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= | ||
| golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= | ||
| golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= | ||
| golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | ||
| golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | ||
| golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs= | ||
| golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | ||
| golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | ||
| golang.org/x/net v0.0.0-20200301022130-244492dfa37a h1:GuSPYbZzB5/dcLNCwLQLsg3obCJtX9IJhpXkvY7kzk0= | ||
| golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= | ||
| golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
| golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= | ||
| golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||
| golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= | ||
| golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= | ||
| golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= | ||
| golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= | ||
| golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= | ||
| golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= | ||
| golang.org/x/tools v0.0.0-20200305224536-de023d59a5d1/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= | ||
| golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= | ||
| layeh.com/gopher-luar v1.0.7 h1:53iv6CCkRs5wyofZ+qVXcyAYQOIG52s6pt4xkqZdq7k= | ||
| layeh.com/gopher-luar v1.0.7/go.mod h1:TPnIVCZ2RJBndm7ohXyaqfhzjlZ+OA2SZR/YwL8tECk= |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| package gluaspider | ||
|
|
||
| import ( | ||
| "strings" | ||
|
|
||
| "github.com/PuerkitoBio/goquery" | ||
| "github.com/go-resty/resty/v2" | ||
| "github.com/yuin/gopher-lua" | ||
| "layeh.com/gopher-luar" | ||
| ) | ||
|
|
||
| // spider type | ||
| type spider struct { | ||
| restyClient *resty.Client | ||
| } | ||
|
|
||
| // NewSpider NewSpider | ||
| func NewSpider() *spider { | ||
| return &spider{ | ||
| restyClient: resty.New(), | ||
| } | ||
| } | ||
|
|
||
| // Get Simple Get Url | ||
| func (s *spider) Get(l *lua.LState) int { | ||
| resp, err := s.restyClient.R().Get(l.CheckString(1)) | ||
|
|
||
| if err != nil { | ||
| l.Push(lua.LNil) | ||
| l.Push(lua.LString(err.Error())) | ||
|
|
||
| return 2 | ||
| } | ||
|
|
||
| return s.newDocumentFromString(l, resp.String()) | ||
| } | ||
|
|
||
| // RestyClient Get RestyClient | ||
| func (s *spider) RestyClient(l *lua.LState) int { | ||
| l.Push(luar.New(l, s.restyClient)) | ||
|
|
||
| return 1 | ||
| } | ||
|
|
||
| // newDocumentFromString New Goquery Document From String | ||
| func (s *spider) newDocumentFromString(l *lua.LState, html string) int { | ||
| doc, err := goquery.NewDocumentFromReader(strings.NewReader(html)) | ||
|
|
||
| if err != nil { | ||
| l.Push(lua.LNil) | ||
| l.Push(lua.LString(err.Error())) | ||
|
|
||
| return 2 | ||
| } | ||
|
|
||
| l.Push(luar.New(l, doc)) | ||
| l.Push(lua.LNil) | ||
|
|
||
| return 2 | ||
| } | ||
|
|
||
| // NewDocumentFromString New Goquery Document From String | ||
| func (s *spider) NewDocumentFromString(l *lua.LState) int { | ||
| return s.newDocumentFromString(l, l.CheckString(1)) | ||
| } | ||
|
|
||
| // Loader Loader | ||
| func (s *spider) Loader(l *lua.LState) int { | ||
| // register functions to the table | ||
| mod := l.SetFuncs(l.NewTable(), map[string]lua.LGFunction{ | ||
| "RestyClient": s.RestyClient, | ||
| "NewDocumentFromString": s.NewDocumentFromString, | ||
| "Get": s.Get, | ||
| }) | ||
|
|
||
| // returns the module | ||
| l.Push(mod) | ||
|
|
||
| return 1 | ||
| } |