diff --git a/Dockerfile b/Dockerfile index 0f899fb..19b2d6f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,27 @@ FROM alpine:3.3 - -ADD . /locations-transformer/ - +ADD *.go .git /locations-transformer/ RUN apk add --update bash \ - && apk --update add git bzr \ - && apk --update add go \ + && apk --update add git go bzr \ + && cd locations-transformer \ + && git fetch origin 'refs/tags/*:refs/tags/*' \ + && BUILDINFO_PACKAGE="github.com/Financial-Times/service-status-go/buildinfo." \ + && VERSION="version=$(git describe --tag --always 2> /dev/null)" \ + && DATETIME="dateTime=$(date -u +%Y%m%d%H%M%S)" \ + && REPOSITORY="repository=$(git config --get remote.origin.url)" \ + && REVISION="revision=$(git rev-parse HEAD)" \ + && BUILDER="builder=$(go version)" \ + && LDFLAGS="-X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \ + && cd .. \ && export GOPATH=/gopath \ && REPO_PATH="github.com/Financial-Times/locations-transformer" \ && mkdir -p $GOPATH/src/${REPO_PATH} \ && cp -r locations-transformer/* $GOPATH/src/${REPO_PATH} \ && cd $GOPATH/src/${REPO_PATH} \ && go get -t ./... \ - && go build \ + && cd $GOPATH/src/${REPO_PATH} \ + && echo ${LDFLAGS} \ + && go build -ldflags="${LDFLAGS}" \ && mv locations-transformer /app \ - && apk del go git bzr \ + && apk del go git bzr\ && rm -rf $GOPATH /var/cache/apk/* CMD [ "/app" ] \ No newline at end of file diff --git a/handlers.go b/handlers.go index 3456859..c0d98d6 100644 --- a/handlers.go +++ b/handlers.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "github.com/Financial-Times/go-fthealth/v1a" log "github.com/Sirupsen/logrus" "github.com/gorilla/mux" "net/http" @@ -12,6 +13,27 @@ type locationsHandler struct { service locationService } +// HealthCheck does something +func (h *locationsHandler) HealthCheck() v1a.Check { + return v1a.Check{ + BusinessImpact: "Unable to respond to request for the location data from TME", + Name: "Check connectivity to TME", + PanicGuide: "https://sites.google.com/a/ft.com/ft-technology-service-transition/home/run-book-library/locations-transfomer", + Severity: 1, + TechnicalSummary: "Cannot connect to TME to be able to supply locations", + Checker: h.checker, + } +} + +// Checker does more stuff +func (h *locationsHandler) checker() (string, error) { + err := h.service.checkConnectivity() + if err == nil { + return "Connectivity to TME is ok", err + } + return "Error connecting to TME", err +} + func newLocationsHandler(service locationService) locationsHandler { return locationsHandler{service: service} } @@ -49,3 +71,10 @@ func writeJSONError(w http.ResponseWriter, errorMsg string, statusCode int) { w.WriteHeader(statusCode) fmt.Fprintln(w, fmt.Sprintf("{\"message\": \"%s\"}", errorMsg)) } + +//GoodToGo returns a 503 if the healthcheck fails - suitable for use from varnish to check availability of a node +func (h *locationsHandler) GoodToGo(writer http.ResponseWriter, req *http.Request) { + if _, err := h.checker(); err != nil { + writer.WriteHeader(http.StatusServiceUnavailable) + } +} diff --git a/handlers_test.go b/handlers_test.go index 4c5ac8b..31831e6 100644 --- a/handlers_test.go +++ b/handlers_test.go @@ -6,12 +6,13 @@ import ( "github.com/stretchr/testify/assert" "net/http" "net/http/httptest" + "strings" "testing" ) const testUUID = "bba39990-c78d-3629-ae83-808c333c6dbc" -const getLocationsResponse = "[{\"apiUrl\":\"http://localhost:8080/transformers/locations/bba39990-c78d-3629-ae83-808c333c6dbc\"}]\n" -const getLocationByUUIDResponse = "{\"uuid\":\"bba39990-c78d-3629-ae83-808c333c6dbc\",\"canonicalName\":\"Metals Markets\",\"tmeIdentifier\":\"MTE3-U3ViamVjdHM=\",\"type\":\"Location\"}\n" +const getLocationsResponse = `[{"apiUrl":"http://localhost:8080/transformers/locations/bba39990-c78d-3629-ae83-808c333c6dbc"}]` +const getLocationByUUIDResponse = `{"uuid":"bba39990-c78d-3629-ae83-808c333c6dbc","alternativeIdentifiers":{"TME":["MTE3-U3ViamVjdHM="],"uuids":["bba39990-c78d-3629-ae83-808c333c6dbc"]},"prefLabel":"SomeLocation","type":"Location"}` func TestHandlers(t *testing.T) { assert := assert.New(t) @@ -23,7 +24,7 @@ func TestHandlers(t *testing.T) { contentType string // Contents of the Content-Type header body string }{ - {"Success - get location by uuid", newRequest("GET", fmt.Sprintf("/transformers/locations/%s", testUUID)), &dummyService{found: true, locations: []location{location{UUID: testUUID, CanonicalName: "Metals Markets", TmeIdentifier: "MTE3-U3ViamVjdHM=", Type: "Location"}}}, http.StatusOK, "application/json", getLocationByUUIDResponse}, + {"Success - get location by uuid", newRequest("GET", fmt.Sprintf("/transformers/locations/%s", testUUID)), &dummyService{found: true, locations: []location{getDummyLocation(testUUID, "SomeLocation", "MTE3-U3ViamVjdHM=")}}, http.StatusOK, "application/json", getLocationByUUIDResponse}, {"Not found - get location by uuid", newRequest("GET", fmt.Sprintf("/transformers/locations/%s", testUUID)), &dummyService{found: false, locations: []location{location{}}}, http.StatusNotFound, "application/json", ""}, {"Success - get locations", newRequest("GET", "/transformers/locations"), &dummyService{found: true, locations: []location{location{UUID: testUUID}}}, http.StatusOK, "application/json", getLocationsResponse}, {"Not found - get locations", newRequest("GET", "/transformers/locations"), &dummyService{found: false, locations: []location{}}, http.StatusNotFound, "application/json", ""}, @@ -33,7 +34,7 @@ func TestHandlers(t *testing.T) { rec := httptest.NewRecorder() router(test.dummyService).ServeHTTP(rec, test.req) assert.True(test.statusCode == rec.Code, fmt.Sprintf("%s: Wrong response code, was %d, should be %d", test.name, rec.Code, test.statusCode)) - assert.Equal(test.body, rec.Body.String(), fmt.Sprintf("%s: Wrong body", test.name)) + assert.Equal(strings.TrimSpace(test.body), strings.TrimSpace(rec.Body.String()), fmt.Sprintf("%s: Wrong body", test.name)) } } @@ -69,3 +70,7 @@ func (s *dummyService) getLocations() ([]locationLink, bool) { func (s *dummyService) getLocationByUUID(uuid string) (location, bool) { return s.locations[0], s.found } + +func (s *dummyService) checkConnectivity() error { + return nil +} diff --git a/location.go b/location.go index c6f27f7..de25f28 100644 --- a/location.go +++ b/location.go @@ -1,18 +1,15 @@ package main type location struct { - UUID string `json:"uuid"` - CanonicalName string `json:"canonicalName"` - TmeIdentifier string `json:"tmeIdentifier"` - Type string `json:"type"` + UUID string `json:"uuid"` + AlternativeIdentifiers alternativeIdentifiers `json:"alternativeIdentifiers,omitempty"` + PrefLabel string `json:"prefLabel"` + Type string `json:"type"` } -type locationVariation struct { - Name string `json:"name"` - Weight string `json:"weight"` - Case string `json:"case"` - Accent string `json:"accent"` - Languages []string `json:"languages"` +type alternativeIdentifiers struct { + TME []string `json:"TME,omitempty"` + Uuids []string `json:"uuids,omitempty"` } type locationLink struct { diff --git a/main.go b/main.go index 5a44d5a..98d7d3c 100644 --- a/main.go +++ b/main.go @@ -1,12 +1,18 @@ package main import ( + "crypto/tls" "fmt" + "github.com/Financial-Times/go-fthealth/v1a" "github.com/Financial-Times/http-handlers-go/httphandlers" + status "github.com/Financial-Times/service-status-go/httphandlers" + "github.com/Financial-Times/tme-reader/tmereader" log "github.com/Sirupsen/logrus" "github.com/gorilla/mux" "github.com/jawher/mow.cli" "github.com/rcrowley/go-metrics" + "github.com/sethgrid/pester" + "net" "net/http" "os" "time" @@ -17,7 +23,7 @@ func init() { } func main() { - app := cli.App("locations-transformer", "A RESTful API for transforming TME Locations to UP json") + app := cli.App("locations-transformer", "A RESTful API for transforming TME locations to UP json") username := app.String(cli.StringOpt{ Name: "tme-username", Value: "", @@ -33,7 +39,7 @@ func main() { token := app.String(cli.StringOpt{ Name: "token", Value: "", - Desc: "Token to be used for accessig TME", + Desc: "Token to be used for accessing TME", EnvVar: "TOKEN", }) baseURL := app.String(cli.StringOpt{ @@ -56,35 +62,74 @@ func main() { }) maxRecords := app.Int(cli.IntOpt{ Name: "maxRecords", - Value: int(DefaultMaxRecords), + Value: int(10000), Desc: "Maximum records to be queried to TME", EnvVar: "MAX_RECORDS", }) slices := app.Int(cli.IntOpt{ Name: "slices", - Value: int(DefaultSlices), + Value: int(10), Desc: "Number of requests to be executed in parallel to TME", EnvVar: "SLICES", }) + tmeTaxonomyName := "GL" + app.Action = func() { - c := &http.Client{ - Timeout: time.Duration(20 * time.Second), - } - s, err := newLocationService(newTmeRepository(c, *tmeBaseURL, *username, *password, *token, *maxRecords, *slices), *baseURL) + client := getResilientClient() + + mf := new(locationTransformer) + s, err := newLocationService(tmereader.NewTmeRepository(client, *tmeBaseURL, *username, *password, *token, *maxRecords, *slices, tmeTaxonomyName, &tmereader.AuthorityFiles{}, mf), *baseURL, tmeTaxonomyName, *maxRecords) if err != nil { log.Errorf("Error while creating LocationsService: [%v]", err.Error()) } + h := newLocationsHandler(s) m := mux.NewRouter() + + // The top one of these feels more correct, but the lower one matches what we have in Dropwizard, + // so it's what apps expect currently same as ping + m.HandleFunc(status.PingPath, status.PingHandler) + m.HandleFunc(status.PingPathDW, status.PingHandler) + m.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler) + m.HandleFunc(status.BuildInfoPathDW, status.BuildInfoHandler) + m.HandleFunc("/__health", v1a.Handler("Locations Transformer Healthchecks", "Checks for accessing TME", h.HealthCheck())) + m.HandleFunc("/__gtg", h.GoodToGo) + m.HandleFunc("/transformers/locations", h.getLocations).Methods("GET") m.HandleFunc("/transformers/locations/{uuid}", h.getLocationByUUID).Methods("GET") + http.Handle("/", m) log.Printf("listening on %d", *port) - http.ListenAndServe(fmt.Sprintf(":%d", *port), + err = http.ListenAndServe(fmt.Sprintf(":%d", *port), httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry, httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), m))) + if err != nil { + log.Errorf("Error by listen and serve: %v", err.Error()) + } + } app.Run(os.Args) } + +func getResilientClient() *pester.Client { + tr := &http.Transport{ + MaxIdleConnsPerHost: 128, + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + } + c := &http.Client{ + Transport: tr, + Timeout: time.Duration(30 * time.Second), + } + client := pester.NewExtendedClient(c) + client.Backoff = pester.ExponentialBackoff + client.MaxRetries = 5 + client.Concurrency = 1 + + return client +} diff --git a/repository.go b/repository.go deleted file mode 100644 index 317b0a2..0000000 --- a/repository.go +++ /dev/null @@ -1,136 +0,0 @@ -package main - -import ( - "encoding/xml" - "fmt" - "io/ioutil" - "net/http" - "sync" -) - -const DefaultMaxRecords = 10000 -const DefaultSlices = 10 -const TaxonomyName = "GL" - -type repository interface { - getLocationsTaxonomy(int) (taxonomy, error) - getSingleLocationTaxonomy(string) (term, error) - MaxRecords() int -} - -type tmeRepository struct { - httpClient httpClient - tmeBaseURL string - accessConfig tmeAccessConfig - maxRecords int - slices int -} - -type tmeAccessConfig struct { - userName string - password string - token string -} - -func (t *tmeRepository) MaxRecords() int { - return t.maxRecords -} - -func newTmeRepository(client httpClient, tmeBaseURL string, userName string, password string, token string, maxRecords int, slices int) repository { - return &tmeRepository{httpClient: client, tmeBaseURL: tmeBaseURL, accessConfig: tmeAccessConfig{userName: userName, password: password, token: token}, maxRecords: maxRecords, slices: slices} -} - -func (t *tmeRepository) getLocationsTaxonomy(startRecord int) (taxonomy, error) { - chunks := t.maxRecords / t.slices - chanResponse := make(chan *response, t.slices) - go func() { - var wg sync.WaitGroup - wg.Add(t.slices) - for i := 0; i < t.slices; i++ { - startPosition := startRecord + i*chunks - - go func(startPosition int) { - tax, err := t.getLocationsInChunks(startPosition, chunks) - - chanResponse <- &response{Taxonomy: tax, Err: err} - wg.Done() - }(startPosition) - } - wg.Wait() - - close(chanResponse) - }() - terms := make([]term, 0, t.maxRecords) - var err error = nil - for resp := range chanResponse { - terms = append(terms, resp.Taxonomy.Terms...) - if resp.Err != nil { - err = resp.Err - } - } - return taxonomy{Terms: terms}, err -} - -func (t *tmeRepository) getLocationsInChunks(startPosition int, maxRecords int) (taxonomy, error) { - req, err := http.NewRequest("GET", fmt.Sprintf("%s/rs/authorityfiles/GL/terms?maximumRecords=%d&startRecord=%d", t.tmeBaseURL, maxRecords, startPosition), nil) - if err != nil { - return taxonomy{}, err - } - req.Header.Add("Accept", "application/xml;charset=utf-8") - req.SetBasicAuth(t.accessConfig.userName, t.accessConfig.password) - req.Header.Add("X-Coco-Auth", fmt.Sprintf("%v", t.accessConfig.token)) - - resp, err := t.httpClient.Do(req) - if err != nil { - return taxonomy{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return taxonomy{}, fmt.Errorf("TME returned %d", resp.StatusCode) - } - - contents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return taxonomy{}, err - } - - tax := taxonomy{} - err = xml.Unmarshal(contents, &tax) - if err != nil { - return taxonomy{}, err - } - return tax, nil -} - -func (t *tmeRepository) getSingleLocationTaxonomy(rawId string) (term, error) { - req, err := http.NewRequest("GET", fmt.Sprintf("%s/rs/authorityfiles/GL/terms/%s", t.tmeBaseURL, rawId), nil) - if err != nil { - return term{}, err - } - req.Header.Add("Accept", "application/xml;charset=utf-8") - req.SetBasicAuth(t.accessConfig.userName, t.accessConfig.password) - req.Header.Add("X-Coco-Auth", fmt.Sprintf("%v", t.accessConfig.token)) - - resp, err := t.httpClient.Do(req) - if err != nil { - return term{}, err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return term{}, fmt.Errorf("TME returned %d HTTP status", resp.StatusCode) - } - - contents, err := ioutil.ReadAll(resp.Body) - if err != nil { - return term{}, err - } - - locationTerm := term{} - err = xml.Unmarshal(contents, &locationTerm) - if err != nil { - return term{}, err - } - return locationTerm, nil -} diff --git a/repository_test.go b/repository_test.go deleted file mode 100644 index 9176fd5..0000000 --- a/repository_test.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "bytes" - "errors" - "fmt" - "github.com/stretchr/testify/assert" - "io/ioutil" - "log" - "net/http" - "os" - "testing" -) - -func TestGetLocationsTaxonomy(t *testing.T) { - assert := assert.New(t) - locationsXML, err := os.Open("sample_locations.xml") - log.Printf("%v\n", err) - tests := []struct { - name string - repo repository - tax taxonomy - err error - }{ - {"Success", repo(dummyClient{assert: assert, tmeBaseURL: "https://test-url.com:40001", - resp: http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(locationsXML)}}), - taxonomy{Terms: []term{ - term{CanonicalName: "Banksville, New York", RawID: "Nstein_GL_US_NY_Municipality_942968"}}}, nil}, - {"Error", repo(dummyClient{assert: assert, tmeBaseURL: "https://test-url.com:40001", - resp: http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(locationsXML)}, err: errors.New("Some error")}), - taxonomy{Terms: []term{}}, errors.New("Some error")}, - {"Non 200 from structure service", repo(dummyClient{assert: assert, tmeBaseURL: "https://test-url.com:40001", - resp: http.Response{StatusCode: http.StatusBadRequest, Body: ioutil.NopCloser(locationsXML)}}), - taxonomy{Terms: []term{}}, errors.New("TME returned 400")}, - {"Unmarshalling error", repo(dummyClient{assert: assert, tmeBaseURL: "https://test-url.com:40001", - resp: http.Response{StatusCode: http.StatusOK, Body: ioutil.NopCloser(bytes.NewReader([]byte("Non xml")))}}), - taxonomy{Terms: []term{}}, errors.New("EOF")}, - } - - for _, test := range tests { - expectedTax, err := test.repo.getLocationsTaxonomy(0) - assert.Equal(test.tax, expectedTax, fmt.Sprintf("%s: Expected taxonomy incorrect", test.name)) - assert.Equal(test.err, err) - } - -} - -func repo(c dummyClient) repository { - return &tmeRepository{httpClient: &c, tmeBaseURL: c.tmeBaseURL, accessConfig: tmeAccessConfig{userName: "test", password: "test", token: "test"}, maxRecords: 1, slices: 1} -} - -type dummyClient struct { - assert *assert.Assertions - resp http.Response - err error - tmeBaseURL string -} - -func (d *dummyClient) Do(req *http.Request) (resp *http.Response, err error) { - d.assert.Contains(req.URL.String(), fmt.Sprintf("%s/rs/authorityfiles/GL/terms?maximumRecords=", d.tmeBaseURL), fmt.Sprintf("Expected url incorrect")) - return &d.resp, d.err -} diff --git a/sample_locations.xml b/sample_locations.xml deleted file mode 100644 index 5eda984..0000000 --- a/sample_locations.xml +++ /dev/null @@ -1,24 +0,0 @@ - - - - Nstein_GL_US_NY_Municipality_942968 - Banksville, New York - true - - - Banksville - 100.0 - New York - - und - - - - - - Nstein_GL_US_NY_County_974157 - BT - - - - \ No newline at end of file diff --git a/service.go b/service.go index 27b9308..e97403b 100644 --- a/service.go +++ b/service.go @@ -1,8 +1,8 @@ package main import ( - "github.com/pborman/uuid" - "log" + "github.com/Financial-Times/tme-reader/tmereader" + log "github.com/Sirupsen/logrus" "net/http" ) @@ -13,18 +13,20 @@ type httpClient interface { type locationService interface { getLocations() ([]locationLink, bool) getLocationByUUID(uuid string) (location, bool) + checkConnectivity() error } type locationServiceImpl struct { - repository repository + repository tmereader.Repository baseURL string - IdMap map[string]string + locationsMap map[string]location locationLinks []locationLink + taxonomyName string + maxTmeRecords int } -func newLocationService(repo repository, baseURL string) (locationService, error) { - - s := &locationServiceImpl{repository: repo, baseURL: baseURL} +func newLocationService(repo tmereader.Repository, baseURL string, taxonomyName string, maxTmeRecords int) (locationService, error) { + s := &locationServiceImpl{repository: repo, baseURL: baseURL, taxonomyName: taxonomyName, maxTmeRecords: maxTmeRecords} err := s.init() if err != nil { return &locationServiceImpl{}, err @@ -33,22 +35,24 @@ func newLocationService(repo repository, baseURL string) (locationService, error } func (s *locationServiceImpl) init() error { - s.IdMap = make(map[string]string) + s.locationsMap = make(map[string]location) responseCount := 0 log.Printf("Fetching locations from TME\n") for { - tax, err := s.repository.getLocationsTaxonomy(responseCount) + terms, err := s.repository.GetTmeTermsFromIndex(responseCount) if err != nil { return err } - if len(tax.Terms) < 1 { + + if len(terms) < 1 { log.Printf("Finished fetching locations from TME\n") break } - s.initLocationsMap(tax.Terms) - responseCount += s.repository.MaxRecords() + s.initLocationsMap(terms) + responseCount += s.maxTmeRecords } log.Printf("Added %d location links\n", len(s.locationLinks)) + return nil } @@ -60,22 +64,24 @@ func (s *locationServiceImpl) getLocations() ([]locationLink, bool) { } func (s *locationServiceImpl) getLocationByUUID(uuid string) (location, bool) { - rawId, found := s.IdMap[uuid] - if !found { - return location{}, false - } - term, err := s.repository.getSingleLocationTaxonomy(rawId) - if err != nil { - return location{}, false - } - return transformLocation(term), true + location, found := s.locationsMap[uuid] + return location, found +} + +func (s *locationServiceImpl) checkConnectivity() error { + // TODO: Can we just hit an endpoint to check if TME is available? Or do we need to make sure we get location taxonmies back? + // _, err := s.repository.GetTmeTermsFromIndex() + // if err != nil { + // return err + // } + return nil } -func (s *locationServiceImpl) initLocationsMap(terms []term) { - for _, t := range terms { - tmeIdentifier := buildTmeIdentifier(t.RawID) - uuid := uuid.NewMD5(uuid.UUID{}, []byte(tmeIdentifier)).String() - s.IdMap[uuid] = t.RawID - s.locationLinks = append(s.locationLinks, locationLink{APIURL: s.baseURL + uuid}) +func (s *locationServiceImpl) initLocationsMap(terms []interface{}) { + for _, iTerm := range terms { + t := iTerm.(term) + top := transformLocation(t, s.taxonomyName) + s.locationsMap[top.UUID] = top + s.locationLinks = append(s.locationLinks, locationLink{APIURL: s.baseURL + top.UUID}) } } diff --git a/service_test.go b/service_test.go index b12055f..329e593 100644 --- a/service_test.go +++ b/service_test.go @@ -12,22 +12,23 @@ func TestGetLocations(t *testing.T) { tests := []struct { name string baseURL string - tax taxonomy + terms []term locations []locationLink found bool err error }{ {"Success", "localhost:8080/transformers/locations/", - taxonomy{Terms: []term{term{CanonicalName: "Banksville, New York", RawID: "Nstein_GL_US_NY_Municipality_942968"}}}, - []locationLink{locationLink{APIURL: "localhost:8080/transformers/locations/095b89cd-4d4c-3195-ba78-e366fbe47291"}}, true, nil}, - {"Error on init", "localhost:8080/transformers/locations/", taxonomy{}, []locationLink(nil), false, errors.New("Error getting taxonomy")}, + []term{term{CanonicalName: "test_location", RawID: "b8337559-ac08-3404-9025-bad51ebe2fc7"}, term{CanonicalName: "Feature", RawID: "mNGQ2MWQ0NDMtMDc5Mi00NWExLTlkMGQtNWZhZjk0NGExOWU2-Z2VucVz"}}, + []locationLink{locationLink{APIURL: "localhost:8080/transformers/locations/e559b6c0-2241-35b9-b970-e55cb8be4cba"}, + locationLink{APIURL: "localhost:8080/transformers/locations/ab4861b5-ba5e-3b67-9871-3bb3e52db103"}}, true, nil}, + {"Error on init", "localhost:8080/transformers/locations/", []term{}, []locationLink(nil), false, errors.New("Error getting taxonomy")}, } for _, test := range tests { - repo := dummyRepo{tax: test.tax, err: test.err} - service, err := newLocationService(&repo, test.baseURL) - actualLocations, found := service.getLocations() - assert.Equal(test.locations, actualLocations, fmt.Sprintf("%s: Expected locations link incorrect", test.name)) + repo := dummyRepo{terms: test.terms, err: test.err} + service, err := newLocationService(&repo, test.baseURL, "Locations", 10000) + expectedLocations, found := service.getLocations() + assert.Equal(test.locations, expectedLocations, fmt.Sprintf("%s: Expected locations link incorrect", test.name)) assert.Equal(test.found, found) assert.Equal(test.err, err) } @@ -37,43 +38,52 @@ func TestGetLocationByUuid(t *testing.T) { assert := assert.New(t) tests := []struct { name string - tax taxonomy + terms []term uuid string location location found bool err error }{ - {"Success", taxonomy{Terms: []term{term{CanonicalName: "Banksville, New York", RawID: "Nstein_GL_US_NY_Municipality_942968"}}}, - "095b89cd-4d4c-3195-ba78-e366fbe47291", location{UUID: "095b89cd-4d4c-3195-ba78-e366fbe47291", CanonicalName: "Banksville, New York", TmeIdentifier: "TnN0ZWluX0dMX1VTX05ZX011bmljaXBhbGl0eV85NDI5Njg=-R0w=", Type: "Location"}, true, nil}, - {"Not found", taxonomy{Terms: []term{term{CanonicalName: "Banksville, New York", RawID: "Nstein_GL_US_NY_Municipality_942968"}}}, + {"Success", []term{term{CanonicalName: "Test_location", RawID: "b8337559-ac08-3404-9025-bad51ebe2fc7"}, term{CanonicalName: "Test_location", RawID: "NGQ2MWQ0NDMtMDc5Mi00NWExLTlkMGQtNWZhZjk0NGExOWU2-Z2VucmVz"}}, + "f7de594e-daa7-3d0e-a997-da4440d0c3b6", getDummyLocation("f7de594e-daa7-3d0e-a997-da4440d0c3b6", "Test_location", "TkdRMk1XUTBORE10TURjNU1pMDBOV0V4TFRsa01HUXROV1poWmprME5HRXhPV1UyLVoyVnVjbVZ6-R0w="), true, nil}, + {"Not found", []term{term{CanonicalName: "Test_location", RawID: "845dc7d7-ae89-4fed-a819-9edcbb3fe507"}, term{CanonicalName: "Feature", RawID: "NGQ2MWdefsdfsfcmVz"}}, "some uuid", location{}, false, nil}, - {"Error on init", taxonomy{}, "some uuid", location{}, false, errors.New("Error getting taxonomy")}, + {"Error on init", []term{}, "some uuid", location{}, false, errors.New("Error getting taxonomy")}, } + for _, test := range tests { - repo := dummyRepo{tax: test.tax, err: test.err} - service, err := newLocationService(&repo, "") - actualLocation, found := service.getLocationByUUID(test.uuid) - assert.Equal(test.location, actualLocation, fmt.Sprintf("%s: Expected location incorrect", test.name)) + repo := dummyRepo{terms: test.terms, err: test.err} + service, err := newLocationService(&repo, "", "GL", 10000) + expectedLocation, found := service.getLocationByUUID(test.uuid) + assert.Equal(test.location, expectedLocation, fmt.Sprintf("%s: Expected location incorrect", test.name)) assert.Equal(test.found, found) assert.Equal(test.err, err) } } type dummyRepo struct { - tax taxonomy - err error + terms []term + err error } -func (d *dummyRepo) getLocationsTaxonomy(startRecord int) (taxonomy, error) { +func (d *dummyRepo) GetTmeTermsFromIndex(startRecord int) ([]interface{}, error) { if startRecord > 0 { - return taxonomy{}, d.err + return nil, d.err + } + var interfaces []interface{} = make([]interface{}, len(d.terms)) + for i, data := range d.terms { + interfaces[i] = data } - return d.tax, d.err + return interfaces, d.err } -func (d *dummyRepo) getSingleLocationTaxonomy(uuid string) (term, error) { - return d.tax.Terms[0], d.err +func (d *dummyRepo) GetTmeTermById(uuid string) (interface{}, error) { + return d.terms[0], d.err } -func (d *dummyRepo) MaxRecords() int { - return 1 +func getDummyLocation(uuid string, prefLabel string, tmeId string) location { + return location{ + UUID: uuid, + PrefLabel: prefLabel, + Type: "Location", + AlternativeIdentifiers: alternativeIdentifiers{TME: []string{tmeId}, Uuids: []string{uuid}}} } diff --git a/taxonomy.go b/taxonomy.go index 1d2b6aa..89657c1 100644 --- a/taxonomy.go +++ b/taxonomy.go @@ -4,13 +4,8 @@ type taxonomy struct { Terms []term `xml:"term"` } -//TODO revise fields later; for now will be used only name and id +//TODO revise fields type term struct { CanonicalName string `xml:"name"` RawID string `xml:"id"` } - -type response struct { - Taxonomy taxonomy - Err error -} diff --git a/transformer.go b/transformer.go index 1662c30..1bc7183 100644 --- a/transformer.go +++ b/transformer.go @@ -2,22 +2,49 @@ package main import ( "encoding/base64" + "encoding/xml" "github.com/pborman/uuid" ) -func transformLocation(t term) location { - tmeIdentifier := buildTmeIdentifier(t.RawID) +func transformLocation(tmeTerm term, taxonomyName string) location { + tmeIdentifier := buildTmeIdentifier(tmeTerm.RawID, taxonomyName) + uuid := uuid.NewMD5(uuid.UUID{}, []byte(tmeIdentifier)).String() return location{ - UUID: uuid.NewMD5(uuid.UUID{}, []byte(tmeIdentifier)).String(), - CanonicalName: t.CanonicalName, - TmeIdentifier: tmeIdentifier, - Type: "Location", + UUID: uuid, + PrefLabel: tmeTerm.CanonicalName, + AlternativeIdentifiers: alternativeIdentifiers{TME: []string{tmeIdentifier}, Uuids: []string{uuid}}, + Type: "Location", } } -func buildTmeIdentifier(rawId string) string { - id := base64.StdEncoding.EncodeToString([]byte(rawId)) - taxonomyName := base64.StdEncoding.EncodeToString([]byte(TaxonomyName)) +func buildTmeIdentifier(rawID string, tmeTermTaxonomyName string) string { + id := base64.StdEncoding.EncodeToString([]byte(rawID)) + taxonomyName := base64.StdEncoding.EncodeToString([]byte(tmeTermTaxonomyName)) return id + "-" + taxonomyName } + +type locationTransformer struct { +} + +func (*locationTransformer) UnMarshallTaxonomy(contents []byte) ([]interface{}, error) { + taxonomy := taxonomy{} + err := xml.Unmarshal(contents, &taxonomy) + if err != nil { + return nil, err + } + interfaces := make([]interface{}, len(taxonomy.Terms)) + for i, d := range taxonomy.Terms { + interfaces[i] = d + } + return interfaces, nil +} + +func (*locationTransformer) UnMarshallTerm(content []byte) (interface{}, error) { + dummyTerm := term{} + err := xml.Unmarshal(content, &dummyTerm) + if err != nil { + return term{}, err + } + return dummyTerm, nil +} diff --git a/transformer_test.go b/transformer_test.go index 1d87142..3ef49da 100644 --- a/transformer_test.go +++ b/transformer_test.go @@ -13,11 +13,21 @@ func TestTransform(t *testing.T) { term term location location }{ - {"Trasform term to location", term{CanonicalName: "Banksville, New York", RawID: "Nstein_GL_US_NY_Municipality_942968"}, location{UUID: "095b89cd-4d4c-3195-ba78-e366fbe47291", CanonicalName: "Banksville, New York", TmeIdentifier: "TnN0ZWluX0dMX1VTX05ZX011bmljaXBhbGl0eV85NDI5Njg=-R0w=", Type: "Location"}}, + {"Transform term to location", term{ + CanonicalName: "Location1", + RawID: "UjB4Zk1UWTBPRE0xLVIyVnVjbVZ6-R0w="}, + location{ + UUID: "6334792f-baf0-3764-8936-fc4f240ca53c", + PrefLabel: "Location1", + AlternativeIdentifiers: alternativeIdentifiers{ + TME: []string{"VWpCNFprMVVXVEJQUkUweExWSXlWblZqYlZaNi1SMHc9-R0w="}, + Uuids: []string{"6334792f-baf0-3764-8936-fc4f240ca53c"}, + }, + Type: "Location"}}, } for _, test := range tests { - expectedLocation := transformLocation(test.term) + expectedLocation := transformLocation(test.term, "GL") assert.Equal(test.location, expectedLocation, fmt.Sprintf("%s: Expected location incorrect", test.name)) }