diff --git a/README.md b/README.md index 9114ad4..3d5ff97 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,27 @@ curl -d '{"url":"https://github.com/open-sauced/insights"}' \ -X POST http://localhost:8080/bake ``` +Alternatively, the /bake-org route can accept a `POST` request with a body that +specifies a GitHub organization, and the server will query the GitHub REST API to +get a list of repositories for the organization and process these repositories. + +Example: + +```bash +curl -d '{"org":"https://github.com/open-sauced"}' \ + -H "Content-Type: application/json" \ + -X POST http://localhost:8080/bake-org +``` + +By default, the server will filter out repositories that are archived before +processing them. However, this can be overridden in the `POST` request in this +way: + +```bash +curl -d '{"org":"https://github.com/open-sauced", "archives": true}' \ + -H "Content-Type: application/json" \ + -X POST http://localhost:8080/bake-org +``` ## 🖥️ Local development diff --git a/go.mod b/go.mod index f7a7d99..cb336a4 100644 --- a/go.mod +++ b/go.mod @@ -4,11 +4,12 @@ go 1.20 require ( github.com/go-git/go-git/v5 v5.6.1 + github.com/google/go-github/v54 v54.0.0 github.com/google/uuid v1.3.1 github.com/joho/godotenv v1.5.1 github.com/lib/pq v1.10.9 go.uber.org/zap v1.24.0 - golang.org/x/sys v0.5.0 + golang.org/x/sys v0.11.0 gopkg.in/yaml.v3 v3.0.1 ) @@ -16,10 +17,12 @@ require ( github.com/Microsoft/go-winio v0.5.2 // indirect github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect github.com/acomagu/bufpipe v1.0.4 // indirect - github.com/cloudflare/circl v1.1.0 // indirect + github.com/cloudflare/circl v1.3.3 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.4.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/imdario/mergo v0.3.13 // indirect github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect @@ -29,7 +32,10 @@ require ( github.com/xanzy/ssh-agent v0.3.3 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.6.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/crypto v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect + google.golang.org/appengine v1.6.7 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect ) diff --git a/go.sum b/go.sum index cdb5eef..2a08d9a 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,9 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/cloudflare/circl v1.1.0 h1:bZgT/A+cikZnKIwn7xL2OBj012Bmvho/o6RpRvv3GKY= github.com/cloudflare/circl v1.1.0/go.mod h1:prBCrKB9DV4poKZY1l9zBXg2QJY7mvgRvtMxxK7fi4I= +github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= +github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -29,8 +30,18 @@ github.com/go-git/go-git-fixtures/v4 v4.3.1 h1:y5z6dd3qi8Hl+stezc8p3JxDkoTRqMAlK github.com/go-git/go-git-fixtures/v4 v4.3.1/go.mod h1:8LHG1a3SRW71ettAD/jW13h8c6AqjVSeL11RAdgaqpo= github.com/go-git/go-git/v5 v5.6.1 h1:q4ZRqQl4pR/ZJHc1L5CFjGA1a10u76aV1iC+nh+bHsk= github.com/go-git/go-git/v5 v5.6.1/go.mod h1:mvyoL6Unz0PiTQrGQfSfiLFhBH1c1e84ylC2MDs4ee8= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-github/v54 v54.0.0 h1:OZdXwow4EAD5jEo5qg+dGFH2DpkyZvVsAehjvJuUL/c= +github.com/google/go-github/v54 v54.0.0/go.mod h1:Sw1LXWHhXRZtzJ9LI5fyJg9wbQzYvFhW8W5P2yaAQ7s= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk= @@ -88,10 +99,12 @@ golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220826181053-bd7e27e6170d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -99,8 +112,11 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220826154423-83b083e8dc8b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -119,26 +135,35 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220825204002-c680a09ffe64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.0.0-20220722155259-a9ba230a4035/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/pkg/clients/github_client.go b/pkg/clients/github_client.go new file mode 100644 index 0000000..14f9b45 --- /dev/null +++ b/pkg/clients/github_client.go @@ -0,0 +1,70 @@ +package clients + +import ( + "context" + "net/http" + + "github.com/google/go-github/v54/github" +) + +type GithubAPIClient struct { + client *github.Client +} + +func NewGithubTokenClient(token string) *GithubAPIClient { + ctx := context.Background() + s := &GithubAPIClient{ + client: github.NewTokenClient(ctx, token), + } + return s +} + +func NewGithubClient(httpClient *http.Client) *GithubAPIClient { + s := &GithubAPIClient{ + client: github.NewClient(httpClient), + } + return s +} + +func (s *GithubAPIClient) ListReposByOrg(org string) ([]*github.Repository, error) { + ctx := context.Background() + opt := &github.RepositoryListByOrgOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + // get all pages of results + var allRepos []*github.Repository + for { + repos, resp, err := s.client.Repositories.ListByOrg(ctx, org, opt) + if err != nil { + return allRepos, err + } + allRepos = append(allRepos, repos...) + if resp.NextPage == 0 { + break + } + opt.Page = resp.NextPage + } + return allRepos, nil +} + +func FilterGithubArchivedRepos(repos []*github.Repository) []*github.Repository { + var filteredRepos []*github.Repository + for _, repo := range repos { + if !*repo.Archived { + filteredRepos = append(filteredRepos, repo) + } + } + return filteredRepos +} + +func GetGithubRepoHTMLUrls(repos []*github.Repository) []string { + var urls []string + for _, repo := range repos { + + htmlURL := repo.GetHTMLURL() + if htmlURL != "" { + urls = append(urls, htmlURL) + } + } + return urls +} diff --git a/pkg/clients/github_client_test.go b/pkg/clients/github_client_test.go new file mode 100644 index 0000000..eb3a891 --- /dev/null +++ b/pkg/clients/github_client_test.go @@ -0,0 +1,50 @@ +package clients + +import ( + "fmt" + "testing" + + "github.com/google/go-github/v54/github" +) + +func createRepoList(org string, totalCount int, archiveCount int) []*github.Repository { + var repoList []*github.Repository + for i := 0; i < totalCount; i++ { + var archiveVal = (i < archiveCount) + var htmlURL = fmt.Sprintf("https://github.com/%s/repo-%d", org, i) + repoList = append(repoList, &github.Repository{ + Archived: &archiveVal, + HTMLURL: &htmlURL, + }) + } + return repoList +} + +func TestFilterGithubArchivedRepos(t *testing.T) { + totalCount := 10 + archiveCount := 2 + filteredCountExpected := (totalCount - archiveCount) + originalRepoList := createRepoList("open-sauced", totalCount, archiveCount) + filteredRepoList := FilterGithubArchivedRepos(originalRepoList) + if len(filteredRepoList) != filteredCountExpected { + t.Errorf("FilteredArchivedRepos() should yield %d items; got %d", filteredCountExpected, len(filteredRepoList)) + } +} + +func TestGetGithubRepoHTMLUrls(t *testing.T) { + expected := []string{ + "https://github.com/open-sauced/repo-0", + "https://github.com/open-sauced/repo-1", + "https://github.com/open-sauced/repo-2", + } + repos := createRepoList("open-sauced", 3, 0) + got := GetGithubRepoHTMLUrls(repos) + if len(expected) != len(got) { + t.Errorf("GetRepoHTMLUrls() should yield count matching input") + } + for i := 0; i < len(got); i++ { + if got[i] != expected[i] { + t.Errorf(`Expected GetRepoHTMLUrls()[%d] to yield "%s"; got "%s"`, i, expected[i], got[i]) + } + } +} diff --git a/pkg/server/server.go b/pkg/server/server.go index 1a8e45d..04c6a92 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -8,7 +8,9 @@ import ( "fmt" "log" "net/http" + "net/url" "strings" + "sync" "sync/atomic" "time" @@ -18,6 +20,7 @@ import ( "github.com/google/uuid" "go.uber.org/zap" + "github.com/open-sauced/pizza/oven/pkg/clients" "github.com/open-sauced/pizza/oven/pkg/common" "github.com/open-sauced/pizza/oven/pkg/database" "github.com/open-sauced/pizza/oven/pkg/insights" @@ -57,7 +60,8 @@ func (p PizzaOvenServer) Run(serverPort string) { //nolint:errcheck defer p.Logger.Sync() p.Logger.Infof("Starting server on port %s", serverPort) - http.HandleFunc("/bake", p.handleRequest) + http.HandleFunc("/bake", p.handleBakeRequest) + http.HandleFunc("/bake-org", p.handleBakeOrgRequest) http.HandleFunc("/ping", p.pingHandler) log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", serverPort), nil)) } @@ -67,7 +71,78 @@ type reqData struct { Wait bool `json:"wait,omitempty"` } -func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { +type orgReqData struct { + Org string `json:"org"` + Wait bool `json:"wait,omitempty"` + Archives bool `json:"archives,omitempty"` +} + +func (p PizzaOvenServer) processOrg(orgURLString string, processArchived bool) ([]string, error) { + + var htmlURLs []string + orgURL, err := url.Parse(orgURLString) + if err != nil { + return htmlURLs, err + } + if orgURL.Hostname() != "github.com" { + return htmlURLs, fmt.Errorf("Cannot parse organizations from %s", orgURL.Hostname()) + } + client := clients.NewGithubClient(nil) + orgName := strings.Trim(orgURL.Path, "/") + repos, err := client.ListReposByOrg(orgName) + if err != nil { + return htmlURLs, fmt.Errorf("Error encountered fetching repositories, org: %s with error: %s ", orgName, err) + } + if !processArchived { + repos = clients.FilterGithubArchivedRepos(repos) + } + htmlURLs = clients.GetGithubRepoHTMLUrls(repos) + return htmlURLs, nil + +} + +/* +Accepts URL value and calls: +normalized = common.NormalizeGitURL() +endpoint = transport.NewEndpoint(normalized) +common.IsValidGitRepo(endpoint) +return endpoint + +returns first encountered error, appropriate HTTP error message, and endpoint string value +*/ +func (p *PizzaOvenServer) getValidRepoEndpoint(repoURL string) (string, string, error) { + p.Logger.Debugf("Validating and normalizing repository URL: %s", repoURL) + normalizedRepoURL, err := common.NormalizeGitURL(repoURL) + if err != nil { + return "", + fmt.Sprintf("Could not normalize provided repo URL: %s", err.Error()), + fmt.Errorf("Could not normalize repo URL %s: %s", repoURL, err.Error()) + } + + repoURLendpoint, err := transport.NewEndpoint(normalizedRepoURL) + if err != nil { + return "", + fmt.Sprintf("Could not create git transport endpoint from provided repo URL: %s", err.Error()), + fmt.Errorf("Could not create git transport endpoint with repo URL %s: %s", repoURL, err.Error()) + } + + ok, err := common.IsValidGitRepo(repoURLendpoint.String()) + if !ok { + if err != nil { + return "", + fmt.Sprintf("Error validating remote git repo URL: %s", err.Error()), + fmt.Errorf("Error validating repo URL %s: %s", repoURL, err.Error()) + } + + return "", + fmt.Sprintf("not valid git repo URL. Expected format protocol://address but got: %s", err.Error()), + fmt.Errorf("Could not validate repo URL %s: %s", repoURL, err.Error()) + } + // Happy Path + return repoURLendpoint.String(), "", nil +} + +func (p PizzaOvenServer) handleBakeRequest(w http.ResponseWriter, r *http.Request) { if r.Method != http.MethodPost { p.Logger.Errorf("Received request with invalid method: %v", r.Body) http.Error(w, "Invalid request method, expected post", http.StatusMethodNotAllowed) @@ -81,51 +156,110 @@ func (p PizzaOvenServer) handleRequest(w http.ResponseWriter, r *http.Request) { http.Error(w, "Could not decode request body", http.StatusBadRequest) return } - - p.Logger.Debugf("Validating and normalizing repository URL: %s", data.URL) - normalizedRepoURL, err := common.NormalizeGitURL(data.URL) - if err != nil { - p.Logger.Debugf("Could not normalize repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("Could not normalize provided repo URL: %s", err.Error()), http.StatusBadRequest) - return - } - - repoURLendpoint, err := transport.NewEndpoint(normalizedRepoURL) + httpErrorString, repoURLendpoint, err := p.getValidRepoEndpoint(data.URL) if err != nil { - p.Logger.Errorf("Could not create git transport endpoint with repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("Could not create git transport endpoint from provided repo URL: %s", err.Error()), http.StatusBadRequest) + p.Logger.Debug(err.Error()) + http.Error(w, httpErrorString, http.StatusBadRequest) return } - - ok, err := common.IsValidGitRepo(repoURLendpoint.String()) - if !ok { + w.WriteHeader(http.StatusAccepted) + if data.Wait { + err = p.processRepository(repoURLendpoint) if err != nil { - p.Logger.Errorf("Error validating repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("Error validating remote git repo URL: %s", err.Error()), http.StatusBadRequest) + p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) return } + } else { + go func() { + err = p.processRepository(repoURLendpoint) + if err != nil { + p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } + }() + } +} - p.Logger.Debug("Could not validate repo URL %s: %s", data.URL, err.Error()) - http.Error(w, fmt.Sprintf("not valid git repo URL. Expected format protocol://address but got: %s", err.Error()), http.StatusBadRequest) +func (p PizzaOvenServer) handleBakeOrgRequest(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + p.Logger.Errorf("Received request with invalid method: %v", r.Body) + http.Error(w, "Invalid request method, expected post", http.StatusMethodNotAllowed) return } + var data orgReqData + err := json.NewDecoder(r.Body).Decode(&data) + if err != nil { + p.Logger.Errorf("Could not decode request json body: %v with error: %v", r.Body, err) + http.Error(w, "Could not decode request body", http.StatusBadRequest) + return + } w.WriteHeader(http.StatusAccepted) + if data.Org == "" { + p.Logger.Error("Could not process blank org input") + http.Error(w, "Could not process input", http.StatusBadRequest) + return + } + cloneURLs, err := p.processOrg(data.Org, data.Archives) + if err != nil { + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) + http.Error(w, "Could not process input", http.StatusInternalServerError) + return + } if data.Wait { - err = p.processRepository(repoURLendpoint.String()) - if err != nil { - p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) + errorChannel := make(chan error) + wg := new(sync.WaitGroup) + wg.Add(len(cloneURLs)) + for _, htmlURL := range cloneURLs { + go func(htmlURL string, wg *sync.WaitGroup, c chan error) { + _, repoURLendpoint, err := p.getValidRepoEndpoint(htmlURL) + if err != nil { + c <- err + } else { + c <- p.processRepository(repoURLendpoint) + } + defer wg.Done() + }(htmlURL, wg, errorChannel) + } + defer wg.Wait() + var errorFound = false + for err := range errorChannel { + if err != nil { + p.Logger.Error(err.Error()) + errorFound = true + } + } + if errorFound { http.Error(w, "Could not process input", http.StatusInternalServerError) return } } else { go func() { - err = p.processRepository(repoURLendpoint.String()) + htmlURLs, err := p.processOrg(data.Org, data.Archives) if err != nil { - p.Logger.Errorf("Could not process repository input: %v with error: %v", r.Body, err) + p.Logger.Errorf("Could not process org input: %v with error: %v", r.Body, err) http.Error(w, "Could not process input", http.StatusInternalServerError) return } + errorChannel := make(chan error) + + for _, htmlURL := range htmlURLs { + go func(htmlURL string, c chan error) { + _, repoURLendpoint, err := p.getValidRepoEndpoint(htmlURL) + if err != nil { + c <- err + } else { + c <- p.processRepository(repoURLendpoint) + } + }(htmlURL, errorChannel) + } + for err := range errorChannel { + if err != nil { + p.Logger.Error(err.Error()) + } + } }() } }