Merge pull request #2 from Financial-Times/unifyIdentifiers

Unify identifiers

Dockerfile

@@ -1,18 +1,27 @@
 FROM alpine:3.3
-
-ADD . /locations-transformer/
-
+ADD *.go .git /locations-transformer/
 RUN apk add --update bash \
-  && apk --update add git bzr \
-  && apk --update add go \
+  && apk --update add git go bzr \
+  && cd locations-transformer \
+  && git fetch origin 'refs/tags/*:refs/tags/*' \
+  && BUILDINFO_PACKAGE="github.com/Financial-Times/service-status-go/buildinfo." \
+  && VERSION="version=$(git describe --tag --always 2> /dev/null)" \
+  && DATETIME="dateTime=$(date -u +%Y%m%d%H%M%S)" \
+  && REPOSITORY="repository=$(git config --get remote.origin.url)" \
+  && REVISION="revision=$(git rev-parse HEAD)" \
+  && BUILDER="builder=$(go version)" \
+  && LDFLAGS="-X '"${BUILDINFO_PACKAGE}$VERSION"' -X '"${BUILDINFO_PACKAGE}$DATETIME"' -X '"${BUILDINFO_PACKAGE}$REPOSITORY"' -X '"${BUILDINFO_PACKAGE}$REVISION"' -X '"${BUILDINFO_PACKAGE}$BUILDER"'" \
+  && cd .. \
   && export GOPATH=/gopath \
   && REPO_PATH="github.com/Financial-Times/locations-transformer" \
   && mkdir -p $GOPATH/src/${REPO_PATH} \
   && cp -r locations-transformer/* $GOPATH/src/${REPO_PATH} \
   && cd $GOPATH/src/${REPO_PATH} \
   && go get -t ./... \
-  && go build \
+  && cd $GOPATH/src/${REPO_PATH} \
+  && echo ${LDFLAGS} \
+  && go build -ldflags="${LDFLAGS}" \
   && mv locations-transformer /app \
-  && apk del go git bzr \
+  && apk del go git bzr\
   && rm -rf $GOPATH /var/cache/apk/*
 CMD [ "/app" ]

handlers.go

@@ -3,6 +3,7 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"github.com/Financial-Times/go-fthealth/v1a"
 	log "github.com/Sirupsen/logrus"
 	"github.com/gorilla/mux"
 	"net/http"
@@ -12,6 +13,27 @@ type locationsHandler struct {
 	service locationService
 }
 
+// HealthCheck does something
+func (h *locationsHandler) HealthCheck() v1a.Check {
+	return v1a.Check{
+		BusinessImpact:   "Unable to respond to request for the location data from TME",
+		Name:             "Check connectivity to TME",
+		PanicGuide:       "https://sites.google.com/a/ft.com/ft-technology-service-transition/home/run-book-library/locations-transfomer",
+		Severity:         1,
+		TechnicalSummary: "Cannot connect to TME to be able to supply locations",
+		Checker:          h.checker,
+	}
+}
+
+// Checker does more stuff
+func (h *locationsHandler) checker() (string, error) {
+	err := h.service.checkConnectivity()
+	if err == nil {
+		return "Connectivity to TME is ok", err
+	}
+	return "Error connecting to TME", err
+}
+
 func newLocationsHandler(service locationService) locationsHandler {
 	return locationsHandler{service: service}
 }
@@ -49,3 +71,10 @@ func writeJSONError(w http.ResponseWriter, errorMsg string, statusCode int) {
 	w.WriteHeader(statusCode)
 	fmt.Fprintln(w, fmt.Sprintf("{\"message\": \"%s\"}", errorMsg))
 }
+
+//GoodToGo returns a 503 if the healthcheck fails - suitable for use from varnish to check availability of a node
+func (h *locationsHandler) GoodToGo(writer http.ResponseWriter, req *http.Request) {
+	if _, err := h.checker(); err != nil {
+		writer.WriteHeader(http.StatusServiceUnavailable)
+	}
+}

handlers_test.go

@@ -6,12 +6,13 @@ import (
 	"github.com/stretchr/testify/assert"
 	"net/http"
 	"net/http/httptest"
+	"strings"
 	"testing"
 )
 
 const testUUID = "bba39990-c78d-3629-ae83-808c333c6dbc"
-const getLocationsResponse = "[{\"apiUrl\":\"http://localhost:8080/transformers/locations/bba39990-c78d-3629-ae83-808c333c6dbc\"}]\n"
-const getLocationByUUIDResponse = "{\"uuid\":\"bba39990-c78d-3629-ae83-808c333c6dbc\",\"canonicalName\":\"Metals Markets\",\"tmeIdentifier\":\"MTE3-U3ViamVjdHM=\",\"type\":\"Location\"}\n"
+const getLocationsResponse = `[{"apiUrl":"http://localhost:8080/transformers/locations/bba39990-c78d-3629-ae83-808c333c6dbc"}]`
+const getLocationByUUIDResponse = `{"uuid":"bba39990-c78d-3629-ae83-808c333c6dbc","alternativeIdentifiers":{"TME":["MTE3-U3ViamVjdHM="],"uuids":["bba39990-c78d-3629-ae83-808c333c6dbc"]},"prefLabel":"SomeLocation","type":"Location"}`
 
 func TestHandlers(t *testing.T) {
 	assert := assert.New(t)
@@ -23,7 +24,7 @@ func TestHandlers(t *testing.T) {
 		contentType  string // Contents of the Content-Type header
 		body         string
 	}{
-		{"Success - get location by uuid", newRequest("GET", fmt.Sprintf("/transformers/locations/%s", testUUID)), &dummyService{found: true, locations: []location{location{UUID: testUUID, CanonicalName: "Metals Markets", TmeIdentifier: "MTE3-U3ViamVjdHM=", Type: "Location"}}}, http.StatusOK, "application/json", getLocationByUUIDResponse},
+		{"Success - get location by uuid", newRequest("GET", fmt.Sprintf("/transformers/locations/%s", testUUID)), &dummyService{found: true, locations: []location{getDummyLocation(testUUID, "SomeLocation", "MTE3-U3ViamVjdHM=")}}, http.StatusOK, "application/json", getLocationByUUIDResponse},
 		{"Not found - get location by uuid", newRequest("GET", fmt.Sprintf("/transformers/locations/%s", testUUID)), &dummyService{found: false, locations: []location{location{}}}, http.StatusNotFound, "application/json", ""},
 		{"Success - get locations", newRequest("GET", "/transformers/locations"), &dummyService{found: true, locations: []location{location{UUID: testUUID}}}, http.StatusOK, "application/json", getLocationsResponse},
 		{"Not found - get locations", newRequest("GET", "/transformers/locations"), &dummyService{found: false, locations: []location{}}, http.StatusNotFound, "application/json", ""},
@@ -33,7 +34,7 @@ func TestHandlers(t *testing.T) {
 		rec := httptest.NewRecorder()
 		router(test.dummyService).ServeHTTP(rec, test.req)
 		assert.True(test.statusCode == rec.Code, fmt.Sprintf("%s: Wrong response code, was %d, should be %d", test.name, rec.Code, test.statusCode))
-		assert.Equal(test.body, rec.Body.String(), fmt.Sprintf("%s: Wrong body", test.name))
+		assert.Equal(strings.TrimSpace(test.body), strings.TrimSpace(rec.Body.String()), fmt.Sprintf("%s: Wrong body", test.name))
 	}
 }
 
@@ -69,3 +70,7 @@ func (s *dummyService) getLocations() ([]locationLink, bool) {
 func (s *dummyService) getLocationByUUID(uuid string) (location, bool) {
 	return s.locations[0], s.found
 }
+
+func (s *dummyService) checkConnectivity() error {
+	return nil
+}

location.go

@@ -1,18 +1,15 @@
 package main
 
 type location struct {
-	UUID          string `json:"uuid"`
-	CanonicalName string `json:"canonicalName"`
-	TmeIdentifier string `json:"tmeIdentifier"`
-	Type          string `json:"type"`
+	UUID                   string                 `json:"uuid"`
+	AlternativeIdentifiers alternativeIdentifiers `json:"alternativeIdentifiers,omitempty"`
+	PrefLabel              string                 `json:"prefLabel"`
+	Type                   string                 `json:"type"`
 }
 
-type locationVariation struct {
-	Name      string   `json:"name"`
-	Weight    string   `json:"weight"`
-	Case      string   `json:"case"`
-	Accent    string   `json:"accent"`
-	Languages []string `json:"languages"`
+type alternativeIdentifiers struct {
+	TME   []string `json:"TME,omitempty"`
+	Uuids []string `json:"uuids,omitempty"`
 }
 
 type locationLink struct {

main.go

@@ -1,12 +1,18 @@
 package main
 
 import (
+	"crypto/tls"
 	"fmt"
+	"github.com/Financial-Times/go-fthealth/v1a"
 	"github.com/Financial-Times/http-handlers-go/httphandlers"
+	status "github.com/Financial-Times/service-status-go/httphandlers"
+	"github.com/Financial-Times/tme-reader/tmereader"
 	log "github.com/Sirupsen/logrus"
 	"github.com/gorilla/mux"
 	"github.com/jawher/mow.cli"
 	"github.com/rcrowley/go-metrics"
+	"github.com/sethgrid/pester"
+	"net"
 	"net/http"
 	"os"
 	"time"
@@ -17,7 +23,7 @@ func init() {
 }
 
 func main() {
-	app := cli.App("locations-transformer", "A RESTful API for transforming TME Locations to UP json")
+	app := cli.App("locations-transformer", "A RESTful API for transforming TME locations to UP json")
 	username := app.String(cli.StringOpt{
 		Name:   "tme-username",
 		Value:  "",
@@ -33,7 +39,7 @@ func main() {
 	token := app.String(cli.StringOpt{
 		Name:   "token",
 		Value:  "",
-		Desc:   "Token to be used for accessig TME",
+		Desc:   "Token to be used for accessing TME",
 		EnvVar: "TOKEN",
 	})
 	baseURL := app.String(cli.StringOpt{
@@ -56,35 +62,74 @@ func main() {
 	})
 	maxRecords := app.Int(cli.IntOpt{
 		Name:   "maxRecords",
-		Value:  int(DefaultMaxRecords),
+		Value:  int(10000),
 		Desc:   "Maximum records to be queried to TME",
 		EnvVar: "MAX_RECORDS",
 	})
 	slices := app.Int(cli.IntOpt{
 		Name:   "slices",
-		Value:  int(DefaultSlices),
+		Value:  int(10),
 		Desc:   "Number of requests to be executed in parallel to TME",
 		EnvVar: "SLICES",
 	})
 
+	tmeTaxonomyName := "GL"
+
 	app.Action = func() {
-		c := &http.Client{
-			Timeout: time.Duration(20 * time.Second),
-		}
-		s, err := newLocationService(newTmeRepository(c, *tmeBaseURL, *username, *password, *token, *maxRecords, *slices), *baseURL)
+		client := getResilientClient()
+
+		mf := new(locationTransformer)
+		s, err := newLocationService(tmereader.NewTmeRepository(client, *tmeBaseURL, *username, *password, *token, *maxRecords, *slices, tmeTaxonomyName, &tmereader.AuthorityFiles{}, mf), *baseURL, tmeTaxonomyName, *maxRecords)
 		if err != nil {
 			log.Errorf("Error while creating LocationsService: [%v]", err.Error())
 		}
+
 		h := newLocationsHandler(s)
 		m := mux.NewRouter()
+
+		// The top one of these feels more correct, but the lower one matches what we have in Dropwizard,
+		// so it's what apps expect currently same as ping
+		m.HandleFunc(status.PingPath, status.PingHandler)
+		m.HandleFunc(status.PingPathDW, status.PingHandler)
+		m.HandleFunc(status.BuildInfoPath, status.BuildInfoHandler)
+		m.HandleFunc(status.BuildInfoPathDW, status.BuildInfoHandler)
+		m.HandleFunc("/__health", v1a.Handler("Locations Transformer Healthchecks", "Checks for accessing TME", h.HealthCheck()))
+		m.HandleFunc("/__gtg", h.GoodToGo)
+
 		m.HandleFunc("/transformers/locations", h.getLocations).Methods("GET")
 		m.HandleFunc("/transformers/locations/{uuid}", h.getLocationByUUID).Methods("GET")
+
 		http.Handle("/", m)
 
 		log.Printf("listening on %d", *port)
-		http.ListenAndServe(fmt.Sprintf(":%d", *port),
+		err = http.ListenAndServe(fmt.Sprintf(":%d", *port),
 			httphandlers.HTTPMetricsHandler(metrics.DefaultRegistry,
 				httphandlers.TransactionAwareRequestLoggingHandler(log.StandardLogger(), m)))
+		if err != nil {
+			log.Errorf("Error by listen and serve: %v", err.Error())
+		}
+
 	}
 	app.Run(os.Args)
 }
+
+func getResilientClient() *pester.Client {
+	tr := &http.Transport{
+		MaxIdleConnsPerHost: 128,
+		TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
+		Dial: (&net.Dialer{
+			Timeout:   30 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).Dial,
+	}
+	c := &http.Client{
+		Transport: tr,
+		Timeout:   time.Duration(30 * time.Second),
+	}
+	client := pester.NewExtendedClient(c)
+	client.Backoff = pester.ExponentialBackoff
+	client.MaxRetries = 5
+	client.Concurrency = 1
+
+	return client
+}